From 7e4bcae7c3f74b870fae0137033a65fdde330d3f Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 24 Apr 2024 10:18:58 -0400 Subject: [PATCH 01/56] Implement software installer storage for S3 and local filesystem (#18493) --- changes/18329-storage-for-software-installers | 1 + cmd/fleet/serve.go | 24 +++++ ee/server/service/mdm_external_test.go | 1 + ee/server/service/service.go | 49 ++++----- .../filesystem/software_installer.go | 99 +++++++++++++++++++ .../filesystem/software_installer_test.go | 84 ++++++++++++++++ server/datastore/s3/software_installer.go | 80 +++++++++++++++ .../datastore/s3/software_installer_test.go | 81 +++++++++++++++ server/datastore/s3/testing_utils.go | 39 +++++--- server/fleet/datastore.go | 4 +- server/fleet/software_installer.go | 34 +++++++ server/service/testing_utils.go | 72 ++++++++------ 12 files changed, 505 insertions(+), 63 deletions(-) create mode 100644 changes/18329-storage-for-software-installers create mode 100644 server/datastore/filesystem/software_installer.go create mode 100644 server/datastore/filesystem/software_installer_test.go create mode 100644 server/datastore/s3/software_installer.go create mode 100644 server/datastore/s3/software_installer_test.go create mode 100644 server/fleet/software_installer.go diff --git a/changes/18329-storage-for-software-installers b/changes/18329-storage-for-software-installers new file mode 100644 index 0000000000..bb091e01cb --- /dev/null +++ b/changes/18329-storage-for-software-installers @@ -0,0 +1 @@ +* Implemented an S3-based and local filesystem-based storage abstraction for software installers. diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index f13e0a453d..108d568707 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -30,6 +30,7 @@ import ( licensectx "github.com/fleetdm/fleet/v4/server/contexts/license" "github.com/fleetdm/fleet/v4/server/cron" "github.com/fleetdm/fleet/v4/server/datastore/cached_mysql" + "github.com/fleetdm/fleet/v4/server/datastore/filesystem" "github.com/fleetdm/fleet/v4/server/datastore/mysql" "github.com/fleetdm/fleet/v4/server/datastore/mysqlredis" "github.com/fleetdm/fleet/v4/server/datastore/redis" @@ -631,6 +632,28 @@ the way that the Fleet server works. if appCfg.MDM.EnabledAndConfigured { profileMatcher = apple_mdm.NewProfileMatcher(redisPool) } + var softwareInstallStore fleet.SoftwareInstallerStore + if config.S3.Bucket != "" { + store, err := s3.NewSoftwareInstallerStore(config.S3) + if err != nil { + initFatal(err, "initializing S3 software installer store") + } + softwareInstallStore = store + level.Info(logger).Log("msg", "using S3 software installer store", "bucket", config.S3.Bucket) + } else { + installerDir := os.TempDir() + if dir := os.Getenv("FLEET_SOFTWARE_INSTALLER_STORE_DIR"); dir != "" { + installerDir = dir + } + store, err := filesystem.NewSoftwareInstallerStore(installerDir) + if err != nil { + level.Error(logger).Log("err", err, "msg", "failed to configure local filesystem software installer store") + softwareInstallStore = fleet.FailingSoftwareInstallerStore{} + } else { + softwareInstallStore = store + level.Info(logger).Log("msg", "using local filesystem software installer store, this is not suitable for production use", "directory", installerDir) + } + } svc, err = eeservice.NewService( svc, @@ -644,6 +667,7 @@ the way that the Fleet server works. mdmPushCertTopic, ssoSessionStore, profileMatcher, + softwareInstallStore, ) if err != nil { initFatal(err, "initial Fleet Premium service") diff --git a/ee/server/service/mdm_external_test.go b/ee/server/service/mdm_external_test.go index e394ee0768..652a23185b 100644 --- a/ee/server/service/mdm_external_test.go +++ b/ee/server/service/mdm_external_test.go @@ -81,6 +81,7 @@ func setupMockDatastorePremiumService() (*mock.Store, *eeservice.Service, contex "", nil, nil, + nil, ) if err != nil { panic(err) diff --git a/ee/server/service/service.go b/ee/server/service/service.go index 0ba18577ae..21dd6ae419 100644 --- a/ee/server/service/service.go +++ b/ee/server/service/service.go @@ -17,17 +17,18 @@ import ( type Service struct { fleet.Service - ds fleet.Datastore - logger kitlog.Logger - config config.FleetConfig - clock clock.Clock - authz *authz.Authorizer - depStorage storage.AllDEPStorage - mdmAppleCommander fleet.MDMAppleCommandIssuer - mdmPushCertTopic string - ssoSessionStore sso.SessionStore - depService *apple_mdm.DEPService - profileMatcher fleet.ProfileMatcher + ds fleet.Datastore + logger kitlog.Logger + config config.FleetConfig + clock clock.Clock + authz *authz.Authorizer + depStorage storage.AllDEPStorage + mdmAppleCommander fleet.MDMAppleCommandIssuer + mdmPushCertTopic string + ssoSessionStore sso.SessionStore + depService *apple_mdm.DEPService + profileMatcher fleet.ProfileMatcher + softwareInstallStore fleet.SoftwareInstallerStore } func NewService( @@ -42,6 +43,7 @@ func NewService( mdmPushCertTopic string, sso sso.SessionStore, profileMatcher fleet.ProfileMatcher, + softwareInstallStore fleet.SoftwareInstallerStore, ) (*Service, error) { authorizer, err := authz.NewAuthorizer() if err != nil { @@ -49,18 +51,19 @@ func NewService( } eeservice := &Service{ - Service: svc, - ds: ds, - logger: logger, - config: config, - clock: c, - authz: authorizer, - depStorage: depStorage, - mdmAppleCommander: mdmAppleCommander, - mdmPushCertTopic: mdmPushCertTopic, - ssoSessionStore: sso, - depService: apple_mdm.NewDEPService(ds, depStorage, logger), - profileMatcher: profileMatcher, + Service: svc, + ds: ds, + logger: logger, + config: config, + clock: c, + authz: authorizer, + depStorage: depStorage, + mdmAppleCommander: mdmAppleCommander, + mdmPushCertTopic: mdmPushCertTopic, + ssoSessionStore: sso, + depService: apple_mdm.NewDEPService(ds, depStorage, logger), + profileMatcher: profileMatcher, + softwareInstallStore: softwareInstallStore, } // Override methods that can't be easily overriden via diff --git a/server/datastore/filesystem/software_installer.go b/server/datastore/filesystem/software_installer.go new file mode 100644 index 0000000000..e8d297fad9 --- /dev/null +++ b/server/datastore/filesystem/software_installer.go @@ -0,0 +1,99 @@ +package filesystem + +import ( + "context" + "io" + "os" + "path/filepath" + + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/fleet" +) + +const softwareInstallersPrefix = "software-installers" + +type installerNotFoundError struct{} + +var _ fleet.NotFoundError = (*installerNotFoundError)(nil) + +func (p installerNotFoundError) Error() string { + return "installer not found" +} + +func (p installerNotFoundError) IsNotFound() bool { + return true +} + +type SoftwareInstallerStore struct { + rootDir string +} + +// NewSoftwareInstallerStore creates a software installer store using the +// local filesystem rooted at the provided rootDir. +func NewSoftwareInstallerStore(rootDir string) (*SoftwareInstallerStore, error) { + // ensure the directories exist (the provided rootDir and the + // softwareInstallersPrefix we create inside it). + dir := filepath.Join(rootDir, softwareInstallersPrefix) + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, err + } + + return &SoftwareInstallerStore{rootDir}, nil +} + +// Get retrieves the requested software installer from the local filesystem. +// It is important that the caller closes the reader when done. +func (i *SoftwareInstallerStore) Get(ctx context.Context, installerID string) (io.ReadCloser, int64, error) { + path := i.pathForInstaller(installerID) + st, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return nil, int64(0), installerNotFoundError{} + } + return nil, 0, ctxerr.Wrap(ctx, err, "retrieving software installer from filesystem store") + } + + sz := st.Size() + f, err := os.Open(path) + if err != nil { + return nil, sz, ctxerr.Wrap(ctx, err, "opening software installer file from filesystem store") + } + return f, sz, nil +} + +// Put stores a software installer in the local filesystem. +func (i *SoftwareInstallerStore) Put(ctx context.Context, installerID string, content io.ReadSeeker) error { + path := i.pathForInstaller(installerID) + + f, err := os.Create(path) + if err != nil { + return ctxerr.Wrap(ctx, err, "creating software installer file in filesystem store") + } + defer f.Close() + + if _, err := io.Copy(f, content); err != nil { + return ctxerr.Wrap(ctx, err, "writing software installer file in filesystem store") + } + if err := f.Close(); err != nil { + return ctxerr.Wrap(ctx, err, "closing software installer file in filesystem store") + } + return nil +} + +// Exists checks if a software installer exists in the filesystem for the ID. +func (i *SoftwareInstallerStore) Exists(ctx context.Context, installerID string) (bool, error) { + path := i.pathForInstaller(installerID) + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, ctxerr.Wrap(ctx, err, "looking up software installer in filesystem store") + } + return true, nil +} + +// pathForInstaller builds local filesystem path to identify the software +// installer. +func (i *SoftwareInstallerStore) pathForInstaller(installerID string) string { + return filepath.Join(i.rootDir, softwareInstallersPrefix, installerID) +} diff --git a/server/datastore/filesystem/software_installer_test.go b/server/datastore/filesystem/software_installer_test.go new file mode 100644 index 0000000000..e3197cbf74 --- /dev/null +++ b/server/datastore/filesystem/software_installer_test.go @@ -0,0 +1,84 @@ +package filesystem + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "io" + "testing" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/stretchr/testify/require" +) + +func TestSoftwareInstaller(t *testing.T) { + ctx := context.Background() + + dir := t.TempDir() + store, err := NewSoftwareInstallerStore(dir) + require.NoError(t, err) + + // get a non-existing installer + blob, length, err := store.Get(ctx, "no-such-installer") + require.Error(t, err) + require.True(t, fleet.IsNotFound(err)) + require.Nil(t, blob) + require.Zero(t, length) + + exists, err := store.Exists(ctx, "no-such-installer") + require.NoError(t, err) + require.False(t, exists) + + createInstallerAndHash := func() ([]byte, string) { + b := make([]byte, 1024) + _, err = rand.Read(b) + require.NoError(t, err) + + h := sha256.New() + _, err = h.Write(b) + require.NoError(t, err) + installerID := hex.EncodeToString(h.Sum(nil)) + + return b, installerID + } + + getAndCheck := func(installerID string, expected []byte) { + rc, sz, err := store.Get(ctx, installerID) + require.NoError(t, err) + require.EqualValues(t, len(expected), sz) + defer rc.Close() + + got, err := io.ReadAll(rc) + require.NoError(t, err) + require.Equal(t, expected, got) + + exists, err := store.Exists(ctx, installerID) + require.NoError(t, err) + require.True(t, exists) + } + + // store an installer + b0, id0 := createInstallerAndHash() + err = store.Put(ctx, id0, bytes.NewReader(b0)) + require.NoError(t, err) + + // read it back, it should match + getAndCheck(id0, b0) + + // store another one + b1, id1 := createInstallerAndHash() + err = store.Put(ctx, id1, bytes.NewReader(b1)) + require.NoError(t, err) + + // read it back, it should match + getAndCheck(id1, b1) + + // replace the first one + err = store.Put(ctx, id0, bytes.NewReader(b0)) + require.NoError(t, err) + + // read it back, it should still match + getAndCheck(id0, b0) +} diff --git a/server/datastore/s3/software_installer.go b/server/datastore/s3/software_installer.go new file mode 100644 index 0000000000..8b3d7462cb --- /dev/null +++ b/server/datastore/s3/software_installer.go @@ -0,0 +1,80 @@ +package s3 + +import ( + "context" + "io" + "path" + + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/fleetdm/fleet/v4/server/config" + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" +) + +const softwareInstallersPrefix = "software-installers" + +// SoftwareInstallerStore implements the fleet.SoftwareInstallerStore to store +// and retrieve software installers from S3. +type SoftwareInstallerStore struct { + *s3store +} + +// NewSoftwareInstallerStore creates a new instance with the given S3 config. +func NewSoftwareInstallerStore(config config.S3Config) (*SoftwareInstallerStore, error) { + s3store, err := newS3store(config) + if err != nil { + return nil, err + } + return &SoftwareInstallerStore{s3store}, nil +} + +// Get retrieves the requested software installer from S3. +// It is important that the caller closes the reader when done. +func (i *SoftwareInstallerStore) Get(ctx context.Context, installerID string) (io.ReadCloser, int64, error) { + key := i.keyForInstaller(installerID) + + req, err := i.s3client.GetObject(&s3.GetObjectInput{Bucket: &i.bucket, Key: &key}) + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + switch aerr.Code() { + case s3.ErrCodeNoSuchKey, s3.ErrCodeNoSuchBucket, "NotFound": + return nil, int64(0), installerNotFoundError{} + } + } + return nil, int64(0), ctxerr.Wrap(ctx, err, "retrieving software installer from S3 store") + } + return req.Body, *req.ContentLength, nil +} + +// Put uploads a software installer to S3. +func (i *SoftwareInstallerStore) Put(ctx context.Context, installerID string, content io.ReadSeeker) error { + key := i.keyForInstaller(installerID) + _, err := i.s3client.PutObject(&s3.PutObjectInput{ + Bucket: &i.bucket, + Body: content, + Key: &key, + }) + return err +} + +// Exists checks if a software installer exists in the S3 bucket for the ID. +func (i *SoftwareInstallerStore) Exists(ctx context.Context, installerID string) (bool, error) { + key := i.keyForInstaller(installerID) + + _, err := i.s3client.HeadObject(&s3.HeadObjectInput{Bucket: &i.bucket, Key: &key}) + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + switch aerr.Code() { + case s3.ErrCodeNoSuchKey, s3.ErrCodeNoSuchBucket, "NotFound": + return false, nil + } + } + return false, ctxerr.Wrap(ctx, err, "checking existence of software installer in S3 store") + } + return true, nil +} + +// keyForInstaller builds an S3 key to identify the software installer. +func (i *SoftwareInstallerStore) keyForInstaller(installerID string) string { + return path.Join(i.prefix, softwareInstallersPrefix, installerID) +} diff --git a/server/datastore/s3/software_installer_test.go b/server/datastore/s3/software_installer_test.go new file mode 100644 index 0000000000..4fb846086e --- /dev/null +++ b/server/datastore/s3/software_installer_test.go @@ -0,0 +1,81 @@ +package s3 + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "io" + "testing" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/stretchr/testify/require" +) + +func TestSoftwareInstaller(t *testing.T) { + ctx := context.Background() + store := SetupTestSoftwareInstallerStore(t, "software-installers-unit-test", "prefix") + + // get a non-existing installer + blob, length, err := store.Get(ctx, "no-such-installer") + require.Error(t, err) + require.True(t, fleet.IsNotFound(err)) + require.Nil(t, blob) + require.Zero(t, length) + + exists, err := store.Exists(ctx, "no-such-installer") + require.NoError(t, err) + require.False(t, exists) + + createInstallerAndHash := func() ([]byte, string) { + b := make([]byte, 1024) + _, err = rand.Read(b) + require.NoError(t, err) + + h := sha256.New() + _, err = h.Write(b) + require.NoError(t, err) + installerID := hex.EncodeToString(h.Sum(nil)) + + return b, installerID + } + + getAndCheck := func(installerID string, expected []byte) { + rc, sz, err := store.Get(ctx, installerID) + require.NoError(t, err) + require.EqualValues(t, len(expected), sz) + defer rc.Close() + + got, err := io.ReadAll(rc) + require.NoError(t, err) + require.Equal(t, expected, got) + + exists, err := store.Exists(ctx, installerID) + require.NoError(t, err) + require.True(t, exists) + } + + // store an installer + b0, id0 := createInstallerAndHash() + err = store.Put(ctx, id0, bytes.NewReader(b0)) + require.NoError(t, err) + + // read it back, it should match + getAndCheck(id0, b0) + + // store another one + b1, id1 := createInstallerAndHash() + err = store.Put(ctx, id1, bytes.NewReader(b1)) + require.NoError(t, err) + + // read it back, it should match + getAndCheck(id1, b1) + + // replace the first one + err = store.Put(ctx, id0, bytes.NewReader(b0)) + require.NoError(t, err) + + // read it back, it should still match + getAndCheck(id0, b0) +} diff --git a/server/datastore/s3/testing_utils.go b/server/datastore/s3/testing_utils.go index 05c4a053fb..50e02f5758 100644 --- a/server/datastore/s3/testing_utils.go +++ b/server/datastore/s3/testing_utils.go @@ -20,12 +20,28 @@ const ( mockInstallerContents = "mock" ) +func SetupTestSoftwareInstallerStore(tb testing.TB, bucket, prefix string) *SoftwareInstallerStore { + store := setupTestStore(tb, bucket, prefix, NewSoftwareInstallerStore) + tb.Cleanup(func() { cleanupStore(tb, store.s3store) }) + return store +} + // SetupTestInstallerStore creates a new store with minio as a back-end // for local testing func SetupTestInstallerStore(tb testing.TB, bucket, prefix string) *InstallerStore { + store := setupTestStore(tb, bucket, prefix, NewInstallerStore) + tb.Cleanup(func() { cleanupStore(tb, store.s3store) }) + return store +} + +type testBucketCreator interface { + CreateTestBucket(name string) error +} + +func setupTestStore[T testBucketCreator](tb testing.TB, bucket, prefix string, newFn func(config.S3Config) (T, error)) T { checkEnv(tb) - store, err := NewInstallerStore(config.S3Config{ + store, err := newFn(config.S3Config{ Bucket: bucket, Prefix: prefix, Region: "minio", @@ -40,8 +56,6 @@ func SetupTestInstallerStore(tb testing.TB, bucket, prefix string) *InstallerSto err = store.CreateTestBucket(bucket) require.NoError(tb, err) - tb.Cleanup(func() { cleanupStore(tb, store) }) - return store } @@ -76,8 +90,9 @@ func mockInstaller(secret, kind string, desktop bool) fleet.Installer { } } -func cleanupStore(tb testing.TB, store *InstallerStore) { +func cleanupStore(tb testing.TB, store *s3store) { checkEnv(tb) + resp, err := store.s3client.ListObjects(&s3.ListObjectsInput{ Bucket: &store.bucket, }) @@ -87,13 +102,15 @@ func cleanupStore(tb testing.TB, store *InstallerStore) { for _, o := range resp.Contents { objs = append(objs, &s3.ObjectIdentifier{Key: o.Key}) } - _, err = store.s3client.DeleteObjects(&s3.DeleteObjectsInput{ - Bucket: &store.bucket, - Delete: &s3.Delete{ - Objects: objs, - }, - }) - require.NoError(tb, err) + if len(objs) > 0 { + _, err = store.s3client.DeleteObjects(&s3.DeleteObjectsInput{ + Bucket: &store.bucket, + Delete: &s3.Delete{ + Objects: objs, + }, + }) + require.NoError(tb, err) + } _, err = store.s3client.DeleteBucket(&s3.DeleteBucketInput{ Bucket: &store.bucket, diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 97f66e6c7f..36c6ff1dd1 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -34,7 +34,9 @@ type CarveStore interface { } // InstallerStore is used to communicate to a blob storage containing pre-built -// fleet-osquery installers +// fleet-osquery installers. This was originally implemented to support the +// Fleet Sandbox and is not expected to be used outside of this: +// https://fleetdm.com/docs/configuration/fleet-server-configuration#packaging type InstallerStore interface { Get(ctx context.Context, installer Installer) (io.ReadCloser, int64, error) Put(ctx context.Context, installer Installer) (string, error) diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go new file mode 100644 index 0000000000..4b4e532511 --- /dev/null +++ b/server/fleet/software_installer.go @@ -0,0 +1,34 @@ +package fleet + +import ( + "context" + "errors" + "io" +) + +// SoftwareInstallerStore is the interface to store and retrieve software +// installer files. Fleet supports storing to the local filesystem and to an +// S3 bucket. +type SoftwareInstallerStore interface { + Get(ctx context.Context, installerID string) (io.ReadCloser, int64, error) + Put(ctx context.Context, installerID string, content io.ReadSeeker) error + Exists(ctx context.Context, installerID string) (bool, error) +} + +// FailingSoftwareInstallerStore is an implementation of SoftwareInstallerStore +// that fails all operations. It is used when S3 is not configured and the +// local filesystem store could not be setup. +type FailingSoftwareInstallerStore struct { +} + +func (FailingSoftwareInstallerStore) Get(ctx context.Context, installerID string) (io.ReadCloser, int64, error) { + return nil, 0, errors.New("software installer store not properly configured") +} + +func (FailingSoftwareInstallerStore) Put(ctx context.Context, installerID string, content io.ReadSeeker) error { + return errors.New("software installer store not properly configured") +} + +func (FailingSoftwareInstallerStore) Exists(ctx context.Context, installerID string) (bool, error) { + return false, errors.New("software installer store not properly configured") +} diff --git a/server/service/testing_utils.go b/server/service/testing_utils.go index 56aeb0df0c..5b7af86800 100644 --- a/server/service/testing_utils.go +++ b/server/service/testing_utils.go @@ -19,6 +19,7 @@ import ( "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/contexts/license" "github.com/fleetdm/fleet/v4/server/datastore/cached_mysql" + "github.com/fleetdm/fleet/v4/server/datastore/filesystem" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/logging" "github.com/fleetdm/fleet/v4/server/mail" @@ -62,11 +63,12 @@ func newTestServiceWithConfig(t *testing.T, ds fleet.Datastore, fleetConfig conf mailer fleet.MailService = &mockMailService{SendEmailFn: func(e fleet.Email) error { return nil }} c clock.Clock = clock.C - is fleet.InstallerStore - mdmStorage fleet.MDMAppleStore - mdmPusher nanomdm_push.Pusher - ssoStore sso.SessionStore - profMatcher fleet.ProfileMatcher + is fleet.InstallerStore + mdmStorage fleet.MDMAppleStore + mdmPusher nanomdm_push.Pusher + ssoStore sso.SessionStore + profMatcher fleet.ProfileMatcher + softwareInstallStore fleet.SoftwareInstallerStore ) if len(opts) > 0 { if opts[0].Clock != nil { @@ -107,6 +109,9 @@ func newTestServiceWithConfig(t *testing.T, ds fleet.Datastore, fleetConfig conf mailer, err = mail.NewService(config.TestConfig()) require.NoError(t, err) } + if opts[0].SoftwareInstallStore != nil { + softwareInstallStore = opts[0].SoftwareInstallStore + } // allow to explicitly set installer store to nil is = opts[0].Is @@ -174,6 +179,15 @@ func newTestServiceWithConfig(t *testing.T, ds fleet.Datastore, fleetConfig conf panic(err) } if lic.IsPremium() { + if softwareInstallStore == nil { + // default to file-based + dir := t.TempDir() + store, err := filesystem.NewSoftwareInstallerStore(dir) + if err != nil { + panic(err) + } + softwareInstallStore = store + } svc, err = eeservice.NewService( svc, ds, @@ -186,6 +200,7 @@ func newTestServiceWithConfig(t *testing.T, ds fleet.Datastore, fleetConfig conf "", ssoStore, profMatcher, + softwareInstallStore, ) if err != nil { panic(err) @@ -275,29 +290,30 @@ func (svc *mockMailService) SendEmail(e fleet.Email) error { type TestNewScheduleFunc func(ctx context.Context, ds fleet.Datastore) fleet.NewCronScheduleFunc type TestServerOpts struct { - Logger kitlog.Logger - License *fleet.LicenseInfo - SkipCreateTestUsers bool - Rs fleet.QueryResultStore - Lq fleet.LiveQueryStore - Pool fleet.RedisPool - FailingPolicySet fleet.FailingPolicySet - Clock clock.Clock - Task *async.Task - EnrollHostLimiter fleet.EnrollHostLimiter - Is fleet.InstallerStore - FleetConfig *config.FleetConfig - MDMStorage fleet.MDMAppleStore - DEPStorage nanodep_storage.AllDEPStorage - SCEPStorage scep_depot.Depot - MDMPusher nanomdm_push.Pusher - HTTPServerConfig *http.Server - StartCronSchedules []TestNewScheduleFunc - UseMailService bool - APNSTopic string - ProfileMatcher fleet.ProfileMatcher - EnableCachedDS bool - NoCacheDatastore bool + Logger kitlog.Logger + License *fleet.LicenseInfo + SkipCreateTestUsers bool + Rs fleet.QueryResultStore + Lq fleet.LiveQueryStore + Pool fleet.RedisPool + FailingPolicySet fleet.FailingPolicySet + Clock clock.Clock + Task *async.Task + EnrollHostLimiter fleet.EnrollHostLimiter + Is fleet.InstallerStore + FleetConfig *config.FleetConfig + MDMStorage fleet.MDMAppleStore + DEPStorage nanodep_storage.AllDEPStorage + SCEPStorage scep_depot.Depot + MDMPusher nanomdm_push.Pusher + HTTPServerConfig *http.Server + StartCronSchedules []TestNewScheduleFunc + UseMailService bool + APNSTopic string + ProfileMatcher fleet.ProfileMatcher + EnableCachedDS bool + NoCacheDatastore bool + SoftwareInstallStore fleet.SoftwareInstallerStore } func RunServerForTestsWithDS(t *testing.T, ds fleet.Datastore, opts ...*TestServerOpts) (map[string]fleet.User, *httptest.Server) { From 845984e3e8d5e6775e1620899b334d9446d98b38 Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:37:35 -0500 Subject: [PATCH 02/56] Update backend `authz` for software installers feature (#18513) --- server/authz/policy.rego | 30 ++++++++++ server/authz/policy_test.go | 95 ++++++++++++++++++++++++++++++ server/fleet/software_installer.go | 16 ++++- 3 files changed, 139 insertions(+), 2 deletions(-) diff --git a/server/authz/policy.rego b/server/authz/policy.rego index c5d742fba5..8e46820688 100644 --- a/server/authz/policy.rego +++ b/server/authz/policy.rego @@ -643,6 +643,36 @@ allow { action == read } +# Global admins, maintainers, observers, and observer_plus can read any software installer. +allow { + object.type == "software_installer" + subject.global_role == [admin, maintainer, observer, observer_plus][_] + action == read +} + +# Global admins, maintainers, and gitops can write any software installer. +allow { + object.type == "software_installer" + subject.global_role == [admin, maintainer, gitops][_] + action == write +} + +# Team admins, maintainers, observers, and observer_plus can read any software installer in their teams. +allow { + not is_null(object.team_id) + object.type == "software_installer" + team_role(subject, object.team_id) == [admin, maintainer, observer, observer_plus][_] + action == read +} + +# Team admins, maintainers, and gitops can write any software installer in their teams. +allow { + not is_null(object.team_id) + object.type == "software_installer" + team_role(subject, object.team_id) == [admin, maintainer, gitops][_] + action == write +} + ## # Apple and Windows MDM ## diff --git a/server/authz/policy_test.go b/server/authz/policy_test.go index e71a954d42..0f2ce9b340 100644 --- a/server/authz/policy_test.go +++ b/server/authz/policy_test.go @@ -500,6 +500,101 @@ func TestAuthorizeSoftwareInventory(t *testing.T) { }) } +func TestAuthorizeSoftwareInstaller(t *testing.T) { + t.Parallel() + + noTeamInstaller := &fleet.SoftwareInstaller{} + team1Installer := &fleet.SoftwareInstaller{TeamID: ptr.Uint(1)} + team2Installer := &fleet.SoftwareInstaller{TeamID: ptr.Uint(2)} + runTestCases(t, []authTestCase{ + {user: nil, object: noTeamInstaller, action: read, allow: false}, + {user: nil, object: noTeamInstaller, action: write, allow: false}, + {user: nil, object: team1Installer, action: read, allow: false}, + {user: nil, object: team1Installer, action: write, allow: false}, + {user: nil, object: team2Installer, action: read, allow: false}, + {user: nil, object: team2Installer, action: write, allow: false}, + + {user: test.UserNoRoles, object: noTeamInstaller, action: read, allow: false}, + {user: test.UserNoRoles, object: noTeamInstaller, action: write, allow: false}, + {user: test.UserNoRoles, object: team1Installer, action: read, allow: false}, + {user: test.UserNoRoles, object: team1Installer, action: write, allow: false}, + {user: test.UserNoRoles, object: team2Installer, action: read, allow: false}, + {user: test.UserNoRoles, object: team2Installer, action: write, allow: false}, + + {user: test.UserAdmin, object: noTeamInstaller, action: read, allow: true}, + {user: test.UserAdmin, object: noTeamInstaller, action: write, allow: true}, + {user: test.UserAdmin, object: team1Installer, action: read, allow: true}, + {user: test.UserAdmin, object: team1Installer, action: write, allow: true}, + {user: test.UserAdmin, object: team2Installer, action: read, allow: true}, + {user: test.UserAdmin, object: team2Installer, action: write, allow: true}, + + {user: test.UserMaintainer, object: noTeamInstaller, action: read, allow: true}, + {user: test.UserMaintainer, object: noTeamInstaller, action: write, allow: true}, + {user: test.UserMaintainer, object: team1Installer, action: read, allow: true}, + {user: test.UserMaintainer, object: team1Installer, action: write, allow: true}, + {user: test.UserMaintainer, object: team2Installer, action: read, allow: true}, + {user: test.UserMaintainer, object: team2Installer, action: write, allow: true}, + + {user: test.UserObserver, object: noTeamInstaller, action: read, allow: true}, + {user: test.UserObserver, object: noTeamInstaller, action: write, allow: false}, + {user: test.UserObserver, object: team1Installer, action: read, allow: true}, + {user: test.UserObserver, object: team1Installer, action: write, allow: false}, + {user: test.UserObserver, object: team2Installer, action: read, allow: true}, + {user: test.UserObserver, object: team2Installer, action: write, allow: false}, + + {user: test.UserObserverPlus, object: noTeamInstaller, action: read, allow: true}, + {user: test.UserObserverPlus, object: noTeamInstaller, action: write, allow: false}, + {user: test.UserObserverPlus, object: team1Installer, action: read, allow: true}, + {user: test.UserObserverPlus, object: team1Installer, action: write, allow: false}, + {user: test.UserObserverPlus, object: team2Installer, action: read, allow: true}, + {user: test.UserObserverPlus, object: team2Installer, action: write, allow: false}, + + // TODO: confirm gitops permissions + {user: test.UserGitOps, object: noTeamInstaller, action: read, allow: false}, + {user: test.UserGitOps, object: noTeamInstaller, action: write, allow: true}, + {user: test.UserGitOps, object: team1Installer, action: read, allow: false}, + {user: test.UserGitOps, object: team1Installer, action: write, allow: true}, + {user: test.UserGitOps, object: team2Installer, action: read, allow: false}, + {user: test.UserGitOps, object: team2Installer, action: write, allow: true}, + + // TODO: confirm gitops permissions + {user: test.UserTeamGitOpsTeam1, object: noTeamInstaller, action: read, allow: false}, + {user: test.UserTeamGitOpsTeam1, object: noTeamInstaller, action: write, allow: false}, + {user: test.UserTeamGitOpsTeam1, object: team1Installer, action: read, allow: false}, + {user: test.UserTeamGitOpsTeam1, object: team1Installer, action: write, allow: true}, + {user: test.UserTeamGitOpsTeam1, object: team2Installer, action: read, allow: false}, + {user: test.UserTeamGitOpsTeam1, object: team2Installer, action: write, allow: false}, + + {user: test.UserTeamAdminTeam1, object: noTeamInstaller, action: read, allow: false}, + {user: test.UserTeamAdminTeam1, object: noTeamInstaller, action: write, allow: false}, + {user: test.UserTeamAdminTeam1, object: team1Installer, action: read, allow: true}, + {user: test.UserTeamAdminTeam1, object: team1Installer, action: write, allow: true}, + {user: test.UserTeamAdminTeam1, object: team2Installer, action: read, allow: false}, + {user: test.UserTeamAdminTeam1, object: team2Installer, action: write, allow: false}, + + {user: test.UserTeamMaintainerTeam1, object: noTeamInstaller, action: read, allow: false}, + {user: test.UserTeamMaintainerTeam1, object: noTeamInstaller, action: write, allow: false}, + {user: test.UserTeamMaintainerTeam1, object: team1Installer, action: read, allow: true}, + {user: test.UserTeamMaintainerTeam1, object: team1Installer, action: write, allow: true}, + {user: test.UserTeamMaintainerTeam1, object: team2Installer, action: read, allow: false}, + {user: test.UserTeamMaintainerTeam1, object: team2Installer, action: write, allow: false}, + + {user: test.UserTeamObserverTeam1, object: noTeamInstaller, action: read, allow: false}, + {user: test.UserTeamObserverTeam1, object: noTeamInstaller, action: write, allow: false}, + {user: test.UserTeamObserverTeam1, object: team1Installer, action: read, allow: true}, + {user: test.UserTeamObserverTeam1, object: team1Installer, action: write, allow: false}, + {user: test.UserTeamObserverTeam1, object: team2Installer, action: read, allow: false}, + {user: test.UserTeamObserverTeam1, object: team2Installer, action: write, allow: false}, + + {user: test.UserTeamObserverPlusTeam1, object: noTeamInstaller, action: read, allow: false}, + {user: test.UserTeamObserverPlusTeam1, object: noTeamInstaller, action: write, allow: false}, + {user: test.UserTeamObserverPlusTeam1, object: team1Installer, action: read, allow: true}, + {user: test.UserTeamObserverPlusTeam1, object: team1Installer, action: write, allow: false}, + {user: test.UserTeamObserverPlusTeam1, object: team2Installer, action: read, allow: false}, + {user: test.UserTeamObserverPlusTeam1, object: team2Installer, action: write, allow: false}, + }) +} + func TestAuthorizeHost(t *testing.T) { t.Parallel() diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index 4b4e532511..07a7af571d 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -18,8 +18,7 @@ type SoftwareInstallerStore interface { // FailingSoftwareInstallerStore is an implementation of SoftwareInstallerStore // that fails all operations. It is used when S3 is not configured and the // local filesystem store could not be setup. -type FailingSoftwareInstallerStore struct { -} +type FailingSoftwareInstallerStore struct{} func (FailingSoftwareInstallerStore) Get(ctx context.Context, installerID string) (io.ReadCloser, int64, error) { return nil, 0, errors.New("software installer store not properly configured") @@ -32,3 +31,16 @@ func (FailingSoftwareInstallerStore) Put(ctx context.Context, installerID string func (FailingSoftwareInstallerStore) Exists(ctx context.Context, installerID string) (bool, error) { return false, errors.New("software installer store not properly configured") } + +// SoftwareInstaller represents a software installer package that can be used to install software on +// hosts in Fleet. +type SoftwareInstaller struct { + // TeamID is the ID of the team. A value of nil means it is scoped to hosts that are assigned to + // no team. + TeamID *uint `json:"team_id"` +} + +// AuthzType implements authz.AuthzTyper. +func (s *SoftwareInstaller) AuthzType() string { + return "software_installer" +} From 28b2593570f4a45c1fa1eace9505d06cd701e77f Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Thu, 25 Apr 2024 08:48:44 -0300 Subject: [PATCH 03/56] add migrations for software installers (#18516) for #18323 # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] If database migrations are included, checked table schema to confirm autoupdate - For database migrations: - [x] Checked schema for all modified table for columns that will auto-update timestamps during migration. - [x] Confirmed that updating the timestamps is acceptable, and will not cause unwanted side effects. - [x] Ensured the correct collation is explicitly set for character columns (`COLLATE utf8mb4_unicode_ci`). --- ...240424124712_AddSoftwareInstallerTables.go | 117 ++++++++++++++++++ server/datastore/mysql/schema.sql | 44 ++++++- 2 files changed, 159 insertions(+), 2 deletions(-) create mode 100644 server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go diff --git a/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go b/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go new file mode 100644 index 0000000000..2c814ae2cb --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go @@ -0,0 +1,117 @@ +package tables + +import ( + "database/sql" + "fmt" +) + +func init() { + MigrationClient.AddMigration(Up_20240424124712, Down_20240424124712) +} + +func Up_20240424124712(tx *sql.Tx) error { + _, err := tx.Exec(` +CREATE TABLE IF NOT EXISTS software_installers ( + id int(10) unsigned NOT NULL AUTO_INCREMENT, + + -- FK to the "software version" this installer matches + software_id bigint(20) unsigned DEFAULT NULL, + + -- Raw osquery SQL statment to be run as a pre-install condition + pre_install_query text COLLATE utf8mb4_unicode_ci DEFAULT NULL, + + -- FK to the script_contents for the script used to install this software + install_script_content_id int(10) unsigned NOT NULL, + + -- FK to the script_contents for the post-script uploaded by the IT admin to + -- be run after the software is installed + post_install_script_content_id int(10) unsigned DEFAULT NULL, + + -- used to track the ID retrieved from the storage containing the installer bytes + storage_id binary(64) NOT NULL, + + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + + PRIMARY KEY (id), + + CONSTRAINT fk_software_installers_version + FOREIGN KEY (software_id) + REFERENCES software (id) + ON DELETE SET NULL + ON UPDATE CASCADE, + + CONSTRAINT fk_software_installers_install_script_content_id + FOREIGN KEY (install_script_content_id) + REFERENCES script_contents (id) + ON DELETE RESTRICT + ON UPDATE CASCADE, + + CONSTRAINT fk_software_installers_post_install_script_content_id + FOREIGN KEY (post_install_script_content_id) + REFERENCES script_contents (id) + ON DELETE RESTRICT + ON UPDATE CASCADE +) + `) + if err != nil { + return fmt.Errorf("creating software_installers table: %w", err) + } + + _, err = tx.Exec(` +-- this table tracks the status of a software installation in a host +CREATE TABLE IF NOT EXISTS host_software_installs ( + id int(10) unsigned NOT NULL AUTO_INCREMENT, + + -- Unique identifier (e.g. UUID) generated for each + -- install run. + execution_id varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + + -- Soft reference to the hosts table, entries in this table are deleted in + -- the application logic when a host is deleted. + host_id int(10) unsigned NOT NULL, + + -- FK to the software installer that's being processed + software_installer_id int(10) unsigned NOT NULL, + + -- Output of the osquery query used to determine if the installer should run. + pre_install_query_output text COLLATE utf8mb4_unicode_ci DEFAULT NULL, + + -- Output of the script used to install the software + install_script_output text COLLATE utf8mb4_unicode_ci DEFAULT NULL, + + -- Exit code of the script used to install the software + install_script_exit_code int(10) DEFAULT NULL, + + -- Output of the post-script run after the software is installed + post_install_script_output text COLLATE utf8mb4_unicode_ci DEFAULT NULL, + + -- Exit code of the post-script run after the software is installed + post_install_script_exit_code int(10) DEFAULT NULL, + + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + uploaded_at timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + PRIMARY KEY (id), + + CONSTRAINT fk_host_software_installs_installer_id + FOREIGN KEY (software_installer_id) + REFERENCES software_installers (id) + ON DELETE CASCADE ON UPDATE CASCADE, + + UNIQUE KEY idx_host_software_installs_host_installer (host_id, software_installer_id), + + -- this index can be used to lookup results for a specific + -- execution (execution ids, e.g. when updating the row for results) + UNIQUE KEY idx_host_software_installs_execution_id (execution_id) +) + `) + if err != nil { + return fmt.Errorf("creating host_software_installs table: %w", err) + } + + return nil +} + +func Down_20240424124712(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 19df87045d..81da4141ee 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -486,6 +486,27 @@ CREATE TABLE `host_software_installed_paths` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; +CREATE TABLE `host_software_installs` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `execution_id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + `host_id` int(10) unsigned NOT NULL, + `software_installer_id` int(10) unsigned NOT NULL, + `pre_install_query_output` text COLLATE utf8mb4_unicode_ci, + `install_script_output` text COLLATE utf8mb4_unicode_ci, + `install_script_exit_code` int(10) DEFAULT NULL, + `post_install_script_output` text COLLATE utf8mb4_unicode_ci, + `post_install_script_exit_code` int(10) DEFAULT NULL, + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `uploaded_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `idx_host_software_installs_host_installer` (`host_id`,`software_installer_id`), + UNIQUE KEY `idx_host_software_installs_execution_id` (`execution_id`), + KEY `fk_host_software_installs_installer_id` (`software_installer_id`), + CONSTRAINT `fk_host_software_installs_installer_id` FOREIGN KEY (`software_installer_id`) REFERENCES `software_installers` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; CREATE TABLE `host_updates` ( `host_id` int(10) unsigned NOT NULL, `software_updated_at` timestamp NULL DEFAULT NULL, @@ -886,9 +907,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=264 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB AUTO_INCREMENT=265 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240424124712,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `mobile_device_management_solutions` ( @@ -1476,6 +1497,25 @@ CREATE TABLE `software_host_counts` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; +CREATE TABLE `software_installers` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `software_id` bigint(20) unsigned DEFAULT NULL, + `pre_install_query` text COLLATE utf8mb4_unicode_ci, + `install_script_content_id` int(10) unsigned NOT NULL, + `post_install_script_content_id` int(10) unsigned DEFAULT NULL, + `storage_id` binary(64) NOT NULL, + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + KEY `fk_software_installers_version` (`software_id`), + KEY `fk_software_installers_install_script_content_id` (`install_script_content_id`), + KEY `fk_software_installers_post_install_script_content_id` (`post_install_script_content_id`), + CONSTRAINT `fk_software_installers_install_script_content_id` FOREIGN KEY (`install_script_content_id`) REFERENCES `script_contents` (`id`) ON UPDATE CASCADE, + CONSTRAINT `fk_software_installers_post_install_script_content_id` FOREIGN KEY (`post_install_script_content_id`) REFERENCES `script_contents` (`id`) ON UPDATE CASCADE, + CONSTRAINT `fk_software_installers_version` FOREIGN KEY (`software_id`) REFERENCES `software` (`id`) ON DELETE SET NULL ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; CREATE TABLE `software_titles` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, From 00d262f5e209ad9c559caf335b111d0db60fd2f4 Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Thu, 25 Apr 2024 09:28:53 -0500 Subject: [PATCH 04/56] Add backend types for software installers feature (#18517) --- server/fleet/software_installer.go | 61 +++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index 07a7af571d..e911b6a70c 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -37,10 +37,69 @@ func (FailingSoftwareInstallerStore) Exists(ctx context.Context, installerID str type SoftwareInstaller struct { // TeamID is the ID of the team. A value of nil means it is scoped to hosts that are assigned to // no team. - TeamID *uint `json:"team_id"` + TeamID *uint `json:"team_id" db:"team_id"` + // Name is the name of the software package. + Name string `json:"name" db:"name"` + // Version is the version of the software package. + Version string `json:"version" db:"version"` + // UploadedAt is the time the software package was uploaded. + UploadedAt string `json:"uploaded_at" db:"uploaded_at"` + // InstallerID is the unique identifier for the software package metadata in Fleet. + InstallerID uint `json:"-" db:"installer_id"` + // InstallScript is the script to run to install the software package. + InstallScript string `json:"install_script" db:"install_script"` + // PreInstallQuery is the query to run as a condition to installing the software package. + PreInstallQuery string `json:"pre_install_query" db:"pre_install_condition"` + // PostInstallScript is the script to run after installing the software package. + PostInstallScript string `json:"post_install_script"` } // AuthzType implements authz.AuthzTyper. func (s *SoftwareInstaller) AuthzType() string { return "software_installer" } + +// SoftwareInstallerStatusSummary represents aggregated status metrics for a software installer package. +type SoftwareInstallerStatusSummary struct { + // Installed is the number of hosts that have the software package installed. + Installed uint `json:"installed" db:"installed"` + // Pending is the number of hosts that have the software package pending installation. + Pending uint `json:"pending" db:"pending"` + // Failed is the number of hosts that have the software package installation failed. + Failed uint `json:"failed" db:"failed"` +} + +// SoftwareInstallerStatus represents the status of a software installer package on a host. +type SoftwareInstallerStatus string + +var ( + SoftwareInstallerPending SoftwareInstallerStatus = "pending" + SoftwareInstallerFailed SoftwareInstallerStatus = "failed" + SoftwareInstallerInstalled SoftwareInstallerStatus = "installed" +) + +// HostSoftwareInstaller represents a software installer package that has been installed on a host. +type HostSoftwareInstallerResult struct { + // InstallUUID is the unique identifier for the software install operation associated with the host. + InstallUUID string `json:"install_uuid" db:"execution_id"` + // SoftwareTitle is the title of the software. + SoftwareTitle string `json:"software_title" db:"software_title"` + // SoftwareVersion is the version of the software. + SoftwareTitleID uint `json:"software_title_id" db:"software_title_id"` + // SoftwarePackage is the name of the software installer package. + SoftwarePackage string `json:"software_package" db:"software_package"` + // HostID is the ID of the host. + HostID uint `json:"host_id" db:"host_id"` + // HostDisplayName is the display name of the host. + HostDisplayName string `json:"host_display_name" db:"host_display_name"` + // Status is the status of the software installer package on the host. + Status SoftwareInstallerStatus `json:"status" db:"status"` + // Detail is the detail of the software installer package on the host. + Detail string `json:"detail" db:"detail"` + // Output is the output of the software installer package on the host. + Output string `json:"output" db:"install_script_output"` + // PreInstallQueryOutput is the output of the pre-install query on the host. + PreInstallQueryOutput string `json:"pre_install_query_output" db:"pre_install_query_output"` + // PostInstallScriptOutput is the output of the post-install script on the host. + PostInstallScriptOutput string `json:"post_install_script_output" db:"post_install_script_output"` +} From 563d55c21801bd09b6ec87bdfc0d225bc4289780 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Mon, 29 Apr 2024 09:13:36 -0400 Subject: [PATCH 05/56] Software installers: extract metadata from installers (part 1) (#18509) --- .../18318-extract-metadata-from-installers | 1 + go.mod | 3 + go.sum | 4 + pkg/file/deb.go | 134 ++++++++++++++++++ pkg/file/file.go | 16 +++ pkg/file/file_test.go | 44 ++++++ pkg/file/pe.go | 45 ++++++ pkg/file/testdata/installers/.gitignore | 6 + pkg/file/xar.go | 115 ++++++++++++++- 9 files changed, 364 insertions(+), 4 deletions(-) create mode 100644 changes/18318-extract-metadata-from-installers create mode 100644 pkg/file/deb.go create mode 100644 pkg/file/pe.go create mode 100644 pkg/file/testdata/installers/.gitignore diff --git a/changes/18318-extract-metadata-from-installers b/changes/18318-extract-metadata-from-installers new file mode 100644 index 0000000000..c504760224 --- /dev/null +++ b/changes/18318-extract-metadata-from-installers @@ -0,0 +1 @@ +* Added support to extract package name and version from software installers. diff --git a/go.mod b/go.mod index 87e3ac429b..4aa773996d 100644 --- a/go.mod +++ b/go.mod @@ -202,6 +202,7 @@ require ( github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/edsrzf/mmap-go v1.1.0 // indirect github.com/elastic/go-sysinfo v1.7.1 // indirect github.com/elastic/go-windows v1.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect @@ -275,6 +276,7 @@ require ( github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/saferwall/pe v1.5.2 // indirect github.com/secure-systems-lab/go-securesystemslib v0.5.0 // indirect github.com/sergi/go-diff v1.2.0 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect @@ -295,6 +297,7 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/yashtewari/glob-intersection v0.1.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect diff --git a/go.sum b/go.sum index b42024dcc0..8dd5d62aee 100644 --- a/go.sum +++ b/go.sum @@ -407,6 +407,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/e-dard/netbug v0.0.0-20151029172837-e64d308a0b20 h1:eDPsdileewX4H5a2Jph4gS8mFf749gzIrzpbnPy1oRs= github.com/e-dard/netbug v0.0.0-20151029172837-e64d308a0b20/go.mod h1:WXFUXJ0Y/SzNqXmhUU7VkE7a2Pag0zZnE2b6I87YWIs= +github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= +github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/elastic/go-licenser v0.4.0/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU= github.com/elastic/go-sysinfo v1.7.1 h1:Wx4DSARcKLllpKT2TnFVdSUJOsybqMYCNQZq1/wO+s0= github.com/elastic/go-sysinfo v1.7.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= @@ -1062,6 +1064,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/saferwall/pe v1.5.2 h1:h5lLtLsyxGHQ9dN6cd8EfeLEBEo5gdqJpkuw4o4vTMY= +github.com/saferwall/pe v1.5.2/go.mod h1:SNzv3cdgk8SBI0UwHfyTcdjawfdnN+nbydnEL7GZ25s= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= github.com/scjalliance/comshim v0.0.0-20230315213746-5e51f40bd3b9 h1:rc/CcqLH3lh8n+csdOuDfP+NuykE0U6AeYSJJHKDgSg= github.com/scjalliance/comshim v0.0.0-20230315213746-5e51f40bd3b9/go.mod h1:a/83NAfUXvEuLpmxDssAXxgUgrEy12MId3Wd7OTs76s= diff --git a/pkg/file/deb.go b/pkg/file/deb.go new file mode 100644 index 0000000000..07f51cf6c7 --- /dev/null +++ b/pkg/file/deb.go @@ -0,0 +1,134 @@ +package file + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/bzip2" + "compress/gzip" + "errors" + "fmt" + "io" + "path" + "path/filepath" + "strings" + + "github.com/blakesmith/ar" + "github.com/xi2/xz" +) + +// ExtractDebMetadata extracts the name and version metadata from a .deb file , +// a debian installer package which is in archive format. +func ExtractDebMetadata(b []byte) (name, version string, err error) { + r := ar.NewReader(bytes.NewReader(b)) + + for { + hdr, err := r.Next() + if err == io.EOF { + break + } else if err != nil { + return "", "", fmt.Errorf("failed to advance to next file in archive: %w", err) + } + + name := path.Clean(hdr.Name) + if strings.HasPrefix(name, "control.tar") { + ext := filepath.Ext(name) + if ext == ".tar" { + ext = "" + } + return parseControl(r, ext) + } + } + + // no control.tar file found, return empty information + return "", "", nil +} + +// parseControl adapted from +// https://github.com/sassoftware/relic/blob/6c510a666832163a5d02587bda8be970d5e29b8c/lib/signdeb/control.go#L38-L39 +// +// Copyright (c) SAS Institute Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Parse basic package info from a control.tar.* stream. +func parseControl(r io.Reader, ext string) (name, version string, err error) { + switch ext { + case ".gz": + gz, err := gzip.NewReader(r) + if err != nil { + return "", "", fmt.Errorf("failed to create gzip reader: %w", err) + } + defer gz.Close() + r = gz + + case ".bz2": + r = bzip2.NewReader(r) + case ".xz": + r, err = xz.NewReader(r, 0) + if err != nil { + return "", "", fmt.Errorf("failed to create xz reader: %w", err) + } + case "": + // uncompressed + default: + return "", "", errors.New("unrecognized compression on control.tar: " + ext) + } + + tr := tar.NewReader(r) + found := false + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } else if err != nil { + return "", "", err + } + if path.Clean(hdr.Name) == "control" { + found = true + break + } + } + + if !found { + return "", "", errors.New("control.tar has no control file") + } + + blob, err := io.ReadAll(tr) + if err != nil { + return "", "", fmt.Errorf("failed to read tar file: %w", err) + } + + scanner := bufio.NewScanner(bytes.NewReader(blob)) + for scanner.Scan() { + line := scanner.Text() + i := strings.IndexAny(line, " \t\r\n") + j := strings.Index(line, ":") + if j < 0 || i < j { + continue + } + + key := line[:j] + value := strings.Trim(line[j+1:], " \t\r\n") + switch strings.ToLower(key) { + case "package": + name = value + case "version": + version = value + } + } + if err := scanner.Err(); err != nil { + return name, version, fmt.Errorf("failed to scan control file: %w", err) + } + return name, version, nil +} diff --git a/pkg/file/file.go b/pkg/file/file.go index c40d6390b4..23745468bb 100644 --- a/pkg/file/file.go +++ b/pkg/file/file.go @@ -11,6 +11,22 @@ import ( "github.com/fleetdm/fleet/v4/pkg/secure" ) +// ExtractInstallerMetadata extracts the software name and version from the +// installer file. The format of the installer is determined based on the +// extension of the filename. +func ExtractInstallerMetadata(filename string, b []byte) (name, version string, err error) { + switch ext := filepath.Ext(filename); ext { + case ".deb": + return ExtractDebMetadata(b) + case ".exe": + return ExtractPEMetadata(b) + case ".pkg": + return ExtractXARMetadata(b) + default: + return "", "", fmt.Errorf("unsupported file type: %s", ext) + } +} + // Copy copies the file from srcPath to dstPath, using the provided permissions. // // Note that on Windows the permissions support is limited in Go's file functions. diff --git a/pkg/file/file_test.go b/pkg/file/file_test.go index f40a84f698..3e0bd9c7b7 100644 --- a/pkg/file/file_test.go +++ b/pkg/file/file_test.go @@ -4,6 +4,7 @@ import ( "io/fs" "os" "path/filepath" + "strings" "testing" "github.com/fleetdm/fleet/v4/pkg/file" @@ -90,3 +91,46 @@ func TestExists(t *testing.T) { require.NoError(t, err) assert.False(t, exists) } + +// TestExtractInstallerMetadata tests the ExtractInstallerMetadata function. It +// calls the function for every file under testdata/installers and checks that +// it returns the expected metadata by comparing it to the software name and +// version in the filename. +// +// The filename should have the following format: +// +// $[$]. +// +// That is, it breaks the file name at the dollar sign and the first part is +// the expected name, the second is the expected version. Note that by default, +// files in testdata/installers are NOT included in git, so the test files must +// be added manually (for size and licenses considerations). Why the dollar +// sign? Because dots, dashes and underlines are more likely to be part of the +// name or version. +func TestExtractInstallerMetadata(t *testing.T) { + dents, err := os.ReadDir(filepath.Join("testdata", "installers")) + if err != nil { + t.Fatal(err) + } + + for _, dent := range dents { + if !dent.Type().IsRegular() || strings.HasPrefix(dent.Name(), ".") { + continue + } + t.Run(dent.Name(), func(t *testing.T) { + parts := strings.Split(strings.TrimSuffix(dent.Name(), filepath.Ext(dent.Name())), "$") + if len(parts) < 2 { + t.Fatalf("invalid filename, expected at least 2 sections, got %d: %s", len(parts), dent.Name()) + } + wantName, wantVersion := parts[0], parts[1] + + content, err := os.ReadFile(filepath.Join("testdata", "installers", dent.Name())) + require.NoError(t, err) + + name, version, err := file.ExtractInstallerMetadata(dent.Name(), content) + require.NoError(t, err) + assert.Equal(t, wantName, name) + assert.Equal(t, wantVersion, version) + }) + } +} diff --git a/pkg/file/pe.go b/pkg/file/pe.go new file mode 100644 index 0000000000..3f6220d264 --- /dev/null +++ b/pkg/file/pe.go @@ -0,0 +1,45 @@ +package file + +import ( + "fmt" + "strings" + + "github.com/saferwall/pe" +) + +// ExtractPEMetadata extracts the name and version metadata from a .exe file in +// the Portable Executable (PE) format. +func ExtractPEMetadata(b []byte) (name, version string, err error) { + // cannot use the "Fast" option, we need the data directories for the + // resources to be available. + pep, err := pe.NewBytes(b, &pe.Options{ + OmitExportDirectory: true, + OmitImportDirectory: true, + OmitExceptionDirectory: true, + OmitSecurityDirectory: true, + OmitRelocDirectory: true, + OmitDebugDirectory: true, + OmitArchitectureDirectory: true, + OmitGlobalPtrDirectory: true, + OmitTLSDirectory: true, + OmitLoadConfigDirectory: true, + OmitBoundImportDirectory: true, + OmitIATDirectory: true, + OmitDelayImportDirectory: true, + OmitCLRHeaderDirectory: true, + }) + if err != nil { + return "", "", fmt.Errorf("error creating PE file: %w", err) + } + defer pep.Close() + + if err := pep.Parse(); err != nil { + return "", "", fmt.Errorf("error parsing PE file: %w", err) + } + + v, err := pep.ParseVersionResources() + if err != nil { + return "", "", fmt.Errorf("error parsing PE version resources: %w", err) + } + return strings.TrimSpace(v["ProductName"]), strings.TrimSpace(v["ProductVersion"]), nil +} diff --git a/pkg/file/testdata/installers/.gitignore b/pkg/file/testdata/installers/.gitignore new file mode 100644 index 0000000000..206053cb6e --- /dev/null +++ b/pkg/file/testdata/installers/.gitignore @@ -0,0 +1,6 @@ +# ignore everything except gitignore +# software installers can be added locally to test the ExtractInstallerMetadata +# logic, but tend to be big binary files with various licenses that might not +# make it possible to include in the repository. +* +!.gitignore diff --git a/pkg/file/xar.go b/pkg/file/xar.go index 71566c0687..0bbe49d278 100644 --- a/pkg/file/xar.go +++ b/pkg/file/xar.go @@ -27,12 +27,17 @@ import ( "errors" "fmt" "io" + "strings" ) -// xarMagic is the [file signature][1] (or magic bytes) for xar -// -// [1]: https://en.wikipedia.org/wiki/List_of_file_signatures -const xarMagic = 0x78617221 +const ( + // xarMagic is the [file signature][1] (or magic bytes) for xar + // + // [1]: https://en.wikipedia.org/wiki/List_of_file_signatures + xarMagic = 0x78617221 + + xarHeaderSize = 28 +) const ( hashNone uint32 = iota @@ -69,6 +74,108 @@ type toc struct { XSignature *any `xml:"x-signature"` } +type xmlXar struct { + XMLName xml.Name `xml:"xar"` + TOC xmlTOC +} + +type xmlTOC struct { + XMLName xml.Name `xml:"toc"` + Files []*xmlFile `xml:"file"` +} + +type xmlFileData struct { + XMLName xml.Name `xml:"data"` + Length int64 `xml:"length"` + Offset int64 `xml:"offset"` + Size int64 `xml:"size"` + Encoding struct { + Style string `xml:"style,attr"` + } `xml:"encoding"` +} + +type xmlFile struct { + XMLName xml.Name `xml:"file"` + Name string `xml:"name"` + Data *xmlFileData +} + +type distributionXML struct { + PkgRef []pkgRef `xml:"pkg-ref"` +} + +type pkgRef struct { + ID string `xml:"id,attr"` + Version string `xml:"version,attr,omitempty"` + Auth string `xml:"auth,attr,omitempty"` + Content string `xml:",chardata"` +} + +// ExtractXARMetadata extracts the name and version metadata from a .pkg file +// in the XAR format. +func ExtractXARMetadata(b []byte) (name, version string, err error) { + var hdr xarHeader + + r := bytes.NewReader(b) + if err := binary.Read(r, binary.BigEndian, &hdr); err != nil { + return "", "", fmt.Errorf("decode xar header: %w", err) + } + + zr, err := zlib.NewReader(io.LimitReader(r, hdr.CompressedSize)) + if err != nil { + return "", "", fmt.Errorf("create zlib reader: %w", err) + } + defer zr.Close() + + var root xmlXar + decoder := xml.NewDecoder(zr) + decoder.Strict = false + if err := decoder.Decode(&root); err != nil { + return "", "", fmt.Errorf("decode xar xml: %w", err) + } + + heapOffset := xarHeaderSize + hdr.CompressedSize + for _, f := range root.TOC.Files { + if f.Name == "Distribution" { + var fileReader io.Reader + heapReader := io.NewSectionReader(r, heapOffset, int64(len(b))-heapOffset) + fileReader = io.NewSectionReader(heapReader, f.Data.Offset, f.Data.Length) + + // the distribution file can be compressed differently than the TOC, the + // actual compression is specified in the Encoding.Style field. + if strings.Contains(f.Data.Encoding.Style, "x-gzip") { + // despite the name, x-gzip fails to decode with the gzip package + // (invalid header), but it works with zlib. + zr, err := zlib.NewReader(fileReader) + if err != nil { + return "", "", fmt.Errorf("create zlib reader: %w", err) + } + defer zr.Close() + fileReader = zr + + // TODO(mna): obviously, we may need to support more decompression methods here... + } + + contents, err := io.ReadAll(fileReader) + if err != nil { + return "", "", fmt.Errorf("reading Distribution file: %w", err) + } + + var distXML distributionXML + if err := xml.Unmarshal(contents, &distXML); err != nil { + return "", "", fmt.Errorf("unmarshal Distribution XML: %w", err) + } + + if len(distXML.PkgRef) > 0 { + return strings.TrimSpace(distXML.PkgRef[0].ID), strings.TrimSpace(distXML.PkgRef[0].Version), nil + } + break + } + } + + return "", "", nil +} + // CheckPKGSignature checks if the provided bytes correspond to a signed pkg // (xar) file. // From ced8e560c9a27c4bc86eca63bd8aa2e3d7939234 Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Mon, 29 Apr 2024 12:22:59 -0500 Subject: [PATCH 06/56] Update software installers schema with reference to software titles (#18589) --- ...240424124712_AddSoftwareInstallerTables.go | 38 +++++++++++++++---- server/datastore/mysql/schema.sql | 17 ++++++--- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go b/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go index 2c814ae2cb..7e7dc7e15f 100644 --- a/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go +++ b/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go @@ -14,8 +14,21 @@ func Up_20240424124712(tx *sql.Tx) error { CREATE TABLE IF NOT EXISTS software_installers ( id int(10) unsigned NOT NULL AUTO_INCREMENT, - -- FK to the "software version" this installer matches - software_id bigint(20) unsigned DEFAULT NULL, + -- team_id NULL is for no team (cannot use 0 with foreign key) + team_id INT(10) UNSIGNED NULL, + -- this field is 0 for global, and the team_id otherwise, and is + -- used for the unique index/constraint (team_id cannot be used + -- as it allows NULL). + global_or_team_id INT(10) UNSIGNED NOT NULL DEFAULT 0, + + -- FK to the "software title" this installer matches + title_id int(10) unsigned DEFAULT NULL, + + -- Filename of the uploaded installer + filename varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + + -- Version extracted from the uploaded installer + version varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, -- Raw osquery SQL statment to be run as a pre-install condition pre_install_query text COLLATE utf8mb4_unicode_ci DEFAULT NULL, @@ -30,13 +43,13 @@ CREATE TABLE IF NOT EXISTS software_installers ( -- used to track the ID retrieved from the storage containing the installer bytes storage_id binary(64) NOT NULL, - created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + uploaded_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id), - CONSTRAINT fk_software_installers_version - FOREIGN KEY (software_id) - REFERENCES software (id) + CONSTRAINT fk_software_installers_title + FOREIGN KEY (title_id) + REFERENCES software_titles (id) ON DELETE SET NULL ON UPDATE CASCADE, @@ -50,7 +63,16 @@ CREATE TABLE IF NOT EXISTS software_installers ( FOREIGN KEY (post_install_script_content_id) REFERENCES script_contents (id) ON DELETE RESTRICT - ON UPDATE CASCADE + ON UPDATE CASCADE, + + CONSTRAINT fk_software_installers_team_id + FOREIGN KEY (team_id) + REFERENCES teams (id) + ON DELETE CASCADE + ON UPDATE CASCADE, + + UNIQUE KEY idx_software_installers_team_id_title_id (global_or_team_id, title_id) + ) `) if err != nil { @@ -89,7 +111,7 @@ CREATE TABLE IF NOT EXISTS host_software_installs ( post_install_script_exit_code int(10) DEFAULT NULL, created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - uploaded_at timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + updated_at timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (id), diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 81da4141ee..783c8be08e 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -497,7 +497,7 @@ CREATE TABLE `host_software_installs` ( `post_install_script_output` text COLLATE utf8mb4_unicode_ci, `post_install_script_exit_code` int(10) DEFAULT NULL, `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - `uploaded_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `idx_host_software_installs_host_installer` (`host_id`,`software_installer_id`), UNIQUE KEY `idx_host_software_installs_execution_id` (`execution_id`), @@ -1499,19 +1499,26 @@ CREATE TABLE `software_host_counts` ( /*!40101 SET character_set_client = utf8 */; CREATE TABLE `software_installers` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `software_id` bigint(20) unsigned DEFAULT NULL, + `team_id` int(10) unsigned DEFAULT NULL, + `global_or_team_id` int(10) unsigned NOT NULL DEFAULT '0', + `title_id` int(10) unsigned DEFAULT NULL, + `filename` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + `version` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, `pre_install_query` text COLLATE utf8mb4_unicode_ci, `install_script_content_id` int(10) unsigned NOT NULL, `post_install_script_content_id` int(10) unsigned DEFAULT NULL, `storage_id` binary(64) NOT NULL, - `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `uploaded_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), - KEY `fk_software_installers_version` (`software_id`), + UNIQUE KEY `idx_software_installers_team_id_title_id` (`global_or_team_id`,`title_id`), + KEY `fk_software_installers_title` (`title_id`), KEY `fk_software_installers_install_script_content_id` (`install_script_content_id`), KEY `fk_software_installers_post_install_script_content_id` (`post_install_script_content_id`), + KEY `fk_software_installers_team_id` (`team_id`), CONSTRAINT `fk_software_installers_install_script_content_id` FOREIGN KEY (`install_script_content_id`) REFERENCES `script_contents` (`id`) ON UPDATE CASCADE, CONSTRAINT `fk_software_installers_post_install_script_content_id` FOREIGN KEY (`post_install_script_content_id`) REFERENCES `script_contents` (`id`) ON UPDATE CASCADE, - CONSTRAINT `fk_software_installers_version` FOREIGN KEY (`software_id`) REFERENCES `software` (`id`) ON DELETE SET NULL ON UPDATE CASCADE + CONSTRAINT `fk_software_installers_team_id` FOREIGN KEY (`team_id`) REFERENCES `teams` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `fk_software_installers_title` FOREIGN KEY (`title_id`) REFERENCES `software_titles` (`id`) ON DELETE SET NULL ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; From 1992561714b41faa01b32a6a95983ed944bc6a55 Mon Sep 17 00:00:00 2001 From: Gabriel Hernandez Date: Tue, 30 Apr 2024 11:42:11 +0100 Subject: [PATCH 07/56] fix go mod conflicts --- go.mod | 6 +++--- go.sum | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index c7b6e28eb8..3906438a15 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/aws/aws-sdk-go v1.44.288 github.com/beevik/etree v1.1.0 github.com/beevik/ntp v0.3.0 + github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb github.com/briandowns/spinner v1.13.0 github.com/cenkalti/backoff v2.2.1+incompatible github.com/cenkalti/backoff/v4 v4.2.1 @@ -87,6 +88,7 @@ require ( github.com/quasilyte/go-ruleguard/dsl v0.3.22 github.com/rs/zerolog v1.20.0 github.com/russellhaering/goxmldsig v1.2.0 + github.com/saferwall/pe v1.5.2 github.com/scjalliance/comshim v0.0.0-20230315213746-5e51f40bd3b9 github.com/sethvargo/go-password v0.2.0 github.com/shirou/gopsutil/v3 v3.23.3 @@ -100,6 +102,7 @@ require ( github.com/tj/assert v0.0.3 github.com/ulikunitz/xz v0.5.10 github.com/urfave/cli/v2 v2.23.5 + github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 github.com/ziutek/mymysql v1.5.4 go.elastic.co/apm/module/apmgorilla/v2 v2.3.0 go.elastic.co/apm/module/apmsql/v2 v2.4.3 @@ -183,7 +186,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 // indirect github.com/aws/smithy-go v1.8.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb // indirect github.com/c-bata/go-prompt v0.2.3 // indirect github.com/caarlos0/ctrlc v1.0.0 // indirect github.com/caarlos0/env/v6 v6.7.0 // indirect @@ -275,7 +277,6 @@ require ( github.com/prometheus/procfs v0.12.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/saferwall/pe v1.5.2 // indirect github.com/secure-systems-lab/go-securesystemslib v0.5.0 // indirect github.com/sergi/go-diff v1.2.0 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect @@ -296,7 +297,6 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/yashtewari/glob-intersection v0.1.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect diff --git a/go.sum b/go.sum index 3763eb862c..e9b0a57a05 100644 --- a/go.sum +++ b/go.sum @@ -991,7 +991,6 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/saferwall/pe v1.5.2 h1:h5lLtLsyxGHQ9dN6cd8EfeLEBEo5gdqJpkuw4o4vTMY= github.com/saferwall/pe v1.5.2/go.mod h1:SNzv3cdgk8SBI0UwHfyTcdjawfdnN+nbydnEL7GZ25s= -github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= github.com/scjalliance/comshim v0.0.0-20230315213746-5e51f40bd3b9 h1:rc/CcqLH3lh8n+csdOuDfP+NuykE0U6AeYSJJHKDgSg= github.com/scjalliance/comshim v0.0.0-20230315213746-5e51f40bd3b9/go.mod h1:a/83NAfUXvEuLpmxDssAXxgUgrEy12MId3Wd7OTs76s= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -1455,6 +1454,7 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From 7d014f9fad55661ffdc58055e4057b2d7af1ef8a Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Tue, 30 Apr 2024 11:20:53 -0400 Subject: [PATCH 08/56] Extract metadata from installers part 2 (#18608) --- go.mod | 111 +++++++++-------- go.sum | 233 ++++++++++++++++++++--------------- pkg/file/deb.go | 29 ++++- pkg/file/file.go | 17 +-- pkg/file/file_test.go | 29 +++-- pkg/file/msi.go | 276 ++++++++++++++++++++++++++++++++++++++++++ pkg/file/pe.go | 19 ++- pkg/file/xar.go | 60 ++++++--- 8 files changed, 579 insertions(+), 195 deletions(-) create mode 100644 pkg/file/msi.go diff --git a/go.mod b/go.mod index 3906438a15..7aa9839931 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/fleetdm/fleet/v4 go 1.21.7 require ( - cloud.google.com/go/pubsub v1.33.0 + cloud.google.com/go/pubsub v1.36.1 fyne.io/systray v1.10.1-0.20240111184411-11c585fff98d github.com/AbGuthrie/goquery/v2 v2.0.1 github.com/DATA-DOG/go-sqlmock v1.5.0 @@ -15,7 +15,7 @@ require ( github.com/andygrunwald/go-jira v1.16.0 github.com/antchfx/xmlquery v1.3.14 github.com/aws/aws-sdk-go v1.44.288 - github.com/beevik/etree v1.1.0 + github.com/beevik/etree v1.3.0 github.com/beevik/ntp v0.3.0 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb github.com/briandowns/spinner v1.13.0 @@ -43,12 +43,12 @@ require ( github.com/go-ole/go-ole v1.2.6 github.com/go-sql-driver/mysql v1.7.1 github.com/gocarina/gocsv v0.0.0-20220310154401-d4df709ca055 - github.com/golang-jwt/jwt/v4 v4.4.2 + github.com/golang-jwt/jwt/v4 v4.5.0 github.com/gomodule/oauth1 v0.2.0 github.com/gomodule/redigo v1.8.9 github.com/google/go-cmp v0.6.0 github.com/google/go-github/v37 v37.0.0 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.6.0 github.com/goreleaser/goreleaser v1.1.0 github.com/goreleaser/nfpm/v2 v2.10.0 github.com/gorilla/mux v1.8.0 @@ -86,21 +86,22 @@ require ( github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_golang v1.19.0 github.com/quasilyte/go-ruleguard/dsl v0.3.22 - github.com/rs/zerolog v1.20.0 + github.com/rs/zerolog v1.32.0 github.com/russellhaering/goxmldsig v1.2.0 github.com/saferwall/pe v1.5.2 + github.com/sassoftware/relic/v7 v7.6.2 github.com/scjalliance/comshim v0.0.0-20230315213746-5e51f40bd3b9 github.com/sethvargo/go-password v0.2.0 github.com/shirou/gopsutil/v3 v3.23.3 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 github.com/spf13/cast v1.4.1 - github.com/spf13/cobra v1.5.0 + github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.10.0 github.com/stretchr/testify v1.8.4 github.com/theupdateframework/go-tuf v0.5.2 github.com/throttled/throttled/v2 v2.8.0 github.com/tj/assert v0.0.3 - github.com/ulikunitz/xz v0.5.10 + github.com/ulikunitz/xz v0.5.11 github.com/urfave/cli/v2 v2.23.5 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 github.com/ziutek/mymysql v1.5.4 @@ -110,54 +111,54 @@ require ( go.etcd.io/bbolt v1.3.6 go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.44.0 - go.opentelemetry.io/otel v1.19.0 + go.opentelemetry.io/otel v1.22.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 - go.opentelemetry.io/otel/sdk v1.19.0 + go.opentelemetry.io/otel/sdk v1.22.0 golang.org/x/crypto v0.22.0 golang.org/x/exp v0.0.0-20230105202349-8879d0199aa3 golang.org/x/image v0.10.0 golang.org/x/mod v0.12.0 golang.org/x/net v0.24.0 golang.org/x/oauth2 v0.16.0 - golang.org/x/sync v0.3.0 + golang.org/x/sync v0.6.0 golang.org/x/sys v0.19.0 golang.org/x/text v0.14.0 golang.org/x/tools v0.13.0 - google.golang.org/api v0.128.0 - google.golang.org/grpc v1.58.3 + google.golang.org/api v0.161.0 + google.golang.org/grpc v1.61.0 gopkg.in/guregu/null.v3 v3.5.0 gopkg.in/ini.v1 v1.67.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v2 v2.4.0 - howett.net/plist v1.0.0 - software.sslmate.com/src/go-pkcs12 v0.0.0-20210415151418-c5206de65a78 + howett.net/plist v1.0.1 + software.sslmate.com/src/go-pkcs12 v0.4.0 ) require ( - cloud.google.com/go v0.110.8 // indirect - cloud.google.com/go/compute v1.23.0 // indirect + cloud.google.com/go v0.112.0 // indirect + cloud.google.com/go/compute v1.23.4 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.2 // indirect - cloud.google.com/go/kms v1.15.2 // indirect - cloud.google.com/go/storage v1.30.1 // indirect + cloud.google.com/go/iam v1.1.6 // indirect + cloud.google.com/go/kms v1.15.6 // indirect + cloud.google.com/go/storage v1.36.0 // indirect code.gitea.io/sdk/gitea v0.15.0 // indirect dario.cat/mergo v1.0.0 // indirect github.com/AlekSi/pointer v1.2.0 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect - github.com/Azure/azure-sdk-for-go v57.0.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect github.com/Azure/azure-storage-blob-go v0.14.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest v0.11.24 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect - github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 // indirect - github.com/Azure/go-autorest/autorest/azure/cli v0.4.3 // indirect + github.com/Azure/go-autorest/autorest v0.11.29 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect + github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 // indirect + github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/DataDog/zstd v1.4.5 // indirect + github.com/DataDog/zstd v1.5.5 // indirect github.com/DisgoOrg/disgohook v1.4.3 // indirect github.com/DisgoOrg/log v1.1.0 // indirect github.com/DisgoOrg/restclient v1.2.7 // indirect @@ -175,16 +176,20 @@ require ( github.com/apex/log v1.9.0 // indirect github.com/armon/go-radix v1.0.0 // indirect github.com/atc0005/go-teams-notify/v2 v2.6.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.9.1 // indirect - github.com/aws/aws-sdk-go-v2/config v1.7.0 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.4.0 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 // indirect - github.com/aws/aws-sdk-go-v2/service/kms v1.5.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 // indirect - github.com/aws/smithy-go v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.24.1 // indirect + github.com/aws/aws-sdk-go-v2/config v1.26.6 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.16.16 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect + github.com/aws/aws-sdk-go-v2/service/kms v1.27.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect + github.com/aws/smithy-go v1.19.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/c-bata/go-prompt v0.2.3 // indirect github.com/caarlos0/ctrlc v1.0.0 // indirect @@ -194,7 +199,7 @@ require ( github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cloudflare/circl v1.3.7 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/dghubble/go-twitter v0.0.0-20210609183100-2fdbf421508e // indirect github.com/dghubble/oauth1 v0.7.0 // indirect @@ -210,27 +215,27 @@ require ( github.com/elastic/go-windows v1.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/fatih/structs v1.1.0 // indirect - github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect - github.com/go-logr/logr v1.2.4 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/glog v1.1.0 // indirect + github.com/golang/glog v1.1.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-github/v39 v39.2.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/rpmpack v0.0.0-20210518075352-dc539ef4f2ea // indirect - github.com/google/s2a-go v0.1.4 // indirect + github.com/google/s2a-go v0.1.7 // indirect github.com/google/wire v0.5.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/goreleaser/chglog v0.1.2 // indirect github.com/goreleaser/fileglob v1.2.0 // indirect @@ -246,19 +251,18 @@ require ( github.com/huandu/xstrings v1.3.2 // indirect github.com/iancoleman/orderedmap v0.2.0 // indirect github.com/imdario/mergo v0.3.12 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.16.5 // indirect github.com/kolide/kit v0.0.0-20221107170827-fb85e3d59eab // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.5 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-ieproxy v0.0.1 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect github.com/mattn/go-tty v0.0.3 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect @@ -266,7 +270,7 @@ require ( github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect + github.com/opencontainers/image-spec v1.1.0-rc6 // indirect github.com/oschwald/maxminddb-golang v1.10.0 // indirect github.com/pelletier/go-toml v1.9.4 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect @@ -303,17 +307,20 @@ require ( go.elastic.co/apm/module/apmhttp/v2 v2.3.0 // indirect go.elastic.co/fastjson v1.1.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel/metric v1.19.0 // indirect - go.opentelemetry.io/otel/trace v1.19.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect + go.opentelemetry.io/otel/metric v1.22.0 // indirect + go.opentelemetry.io/otel/trace v1.22.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect + go.uber.org/goleak v1.3.0 // indirect gocloud.dev v0.24.0 // indirect golang.org/x/term v0.19.0 // indirect - golang.org/x/time v0.3.0 // indirect + golang.org/x/time v0.5.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20240205150955-31a09d347014 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/mail.v2 v2.3.1 // indirect diff --git a/go.sum b/go.sum index e9b0a57a05..844225fe39 100644 --- a/go.sum +++ b/go.sum @@ -31,35 +31,35 @@ cloud.google.com/go v0.92.2/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y cloud.google.com/go v0.92.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= cloud.google.com/go v0.94.0/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME= -cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= +cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= +cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= -cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute v1.23.4 h1:EBT9Nw4q3zyE7G45Wvv3MzolIrCJEuHys5muLY0wvAw= +cloud.google.com/go/compute v1.23.4/go.mod h1:/EJMj55asU6kAFnuZET8zqgwgJ9FvXWXOkkfQZa4ioI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/firestore v1.5.0/go.mod h1:c4nNYR1qdq7eaZ+jSc5fonrQN2k3M7sWATcYTiakjEo= -cloud.google.com/go/iam v1.1.2 h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4= -cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= +cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= cloud.google.com/go/kms v0.1.0/go.mod h1:8Qp8PCAypHg4FdmlyW1QRAv09BGQ9Uzh7JnmIZxPk+c= -cloud.google.com/go/kms v1.15.2 h1:lh6qra6oC4AyWe5fUUUBe/S27k12OHAleOOOw6KakdE= -cloud.google.com/go/kms v1.15.2/go.mod h1:3hopT4+7ooWRCjc2DxgnpESFxhIraaI2IpAVUEhbT/w= +cloud.google.com/go/kms v1.15.6 h1:ktpEMQmsOAYj3VZwH020FcQlm23BVYg8T8O1woG2GcE= +cloud.google.com/go/kms v1.15.6/go.mod h1:yF75jttnIdHfGBoE51AKsD/Yqf+/jICzB9v1s1acsms= cloud.google.com/go/monitoring v0.1.0/go.mod h1:Hpm3XfzJv+UTiXzCG5Ffp0wijzHTC7Cv4eR7o3x/fEE= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/pubsub v1.16.0/go.mod h1:6A8EfoWZ/lUvCWStKGwAWauJZSiuV0Mkmu6WilK/TxQ= -cloud.google.com/go/pubsub v1.33.0 h1:6SPCPvWav64tj0sVX/+npCBKhUi/UjJehy9op/V3p2g= -cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= +cloud.google.com/go/pubsub v1.36.1 h1:dfEPuGCHGbWUhaMCTHUFjfroILEkx55iUmKBZTP5f+Y= +cloud.google.com/go/pubsub v1.36.1/go.mod h1:iYjCa9EzWOoBiTdd4ps7QoMtMln5NwaZQpK1hbRfBDE= cloud.google.com/go/secretmanager v0.1.0/go.mod h1:3nGKHvnzDUVit7U0S9KAKJ4aOsO1xtwRG+7ey5LK1bM= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= @@ -67,8 +67,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.16.1/go.mod h1:LaNorbty3ehnU3rEjXSNV/NRgQA0O8Y+uh6bPe5UOk4= -cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= -cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= +cloud.google.com/go/storage v1.36.0 h1:P0mOkAcaJxhCTvAkMhxMfrTKiNcub4YmmPBtlhAyTr8= +cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= cloud.google.com/go/trace v0.1.0/go.mod h1:wxEwsoeRVPbeSkt7ZC9nWCgmoKQRAoySN7XHW2AmI7g= code.gitea.io/gitea-vet v0.2.1/go.mod h1:zcNbT/aJEmivCAhfmkHOlT645KNOf9W2KnkLgFjGGfE= code.gitea.io/sdk/gitea v0.15.0 h1:tsNhxDM/2N1Ohv1Xq5UWrht/esg0WmtRj4wsHVHriTg= @@ -90,8 +90,9 @@ github.com/Azure/azure-amqp-common-go/v3 v3.1.1/go.mod h1:YsDaPfaO9Ub2XeSKdIy2Df github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= github.com/Azure/azure-sdk-for-go v51.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v57.0.0+incompatible h1:isVki3PbIFrwKvKdVP1byxo73/pt+Nn174YxW1k4PNw= github.com/Azure/azure-sdk-for-go v57.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-service-bus-go v0.10.16/go.mod h1:MlkLwGGf1ewcx5jZadn0gUEty+tTg0RaElr6bPf+QhI= github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= @@ -106,26 +107,33 @@ github.com/Azure/go-autorest/autorest v0.11.3/go.mod h1:JFgpikqFJ/MleTTxwepExTKn github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest v0.11.20/go.mod h1:o3tqFY+QR40VOlk+pV4d77mORO64jOXSgEnPQgLK6JY= -github.com/Azure/go-autorest/autorest v0.11.24 h1:1fIGgHKqVm54KIPT+q8Zmd1QlVsmHqeUGso5qm2BqqE= github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= +github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= +github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.14/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.15/go.mod h1:tGMin8I49Yij6AQ+rvV+Xa/zwxYQB5hmsd6DkfAx2+A= -github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQrvuza2bwSnNdhmzLfQ= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= -github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 h1:TzPg6B6fTZ0G1zBf3T54aI7p3cAT6u//TOXGPmFMOXg= +github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= +github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8= +github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c= github.com/Azure/go-autorest/autorest/azure/auth v0.5.8/go.mod h1:kxyKZTSfKh8OVFWPAgOgQ/frrJgeYQJPyR5fLFmXko4= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 h1:wkAZRgT/pn8HhFyzfe9UnqOjJYqlembgCTi72Bm/xKk= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.12/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= -github.com/Azure/go-autorest/autorest/azure/cli v0.4.3 h1:DOhB+nXkF7LN0JfBGB5YtCF6QLK8mLe4psaHF7ZQEKM= github.com/Azure/go-autorest/autorest/azure/cli v0.4.3/go.mod h1:yAQ2b6eP/CmLPnmLvxtT1ALIY3OR1oFcCqVBi8vHiTc= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= @@ -142,8 +150,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= -github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= +github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/DisgoOrg/disgohook v1.4.3 h1:JtZiV0jAku9NZRYD6wVH7tWY1617rh4tRqn4ihTUJRc= github.com/DisgoOrg/disgohook v1.4.3/go.mod h1:aHNyBHq1pBbdWrkCq3ZCSBeavUoGWZAAT4+609EcrvU= github.com/DisgoOrg/log v1.1.0 h1:a6hLfVSDuTFJc5AKQ8FDYQ5TASnwk3tciUyXThm1CR4= @@ -240,31 +248,49 @@ github.com/aws/aws-sdk-go v1.40.34/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm github.com/aws/aws-sdk-go v1.44.288 h1:Ln7fIao/nl0ACtelgR1I4AiEw/GLNkKcXfCaHupUW5Q= github.com/aws/aws-sdk-go v1.44.288/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= -github.com/aws/aws-sdk-go-v2 v1.9.1 h1:ZbovGV/qo40nrOJ4q8G33AGICzaPI45FHQWJ9650pF4= -github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= -github.com/aws/aws-sdk-go-v2/config v1.7.0 h1:J2cZ7qe+3IpqBEXnHUrFrOjoB9BlsXg7j53vxcl5IVg= +github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU= +github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= github.com/aws/aws-sdk-go-v2/config v1.7.0/go.mod h1:w9+nMZ7soXCe5nT46Ri354SNhXDQ6v+V5wqDjnZE+GY= -github.com/aws/aws-sdk-go-v2/credentials v1.4.0 h1:kmvesfjY861FzlCU9mvAfe01D9aeXcG2ZuC+k9F2YLM= +github.com/aws/aws-sdk-go-v2/config v1.26.6 h1:Z/7w9bUqlRI0FFQpetVuFYEsjzE3h7fpU6HuGmfPL/o= +github.com/aws/aws-sdk-go-v2/config v1.26.6/go.mod h1:uKU6cnDmYCvJ+pxO9S4cWDb2yWWIH5hra+32hVh1MI4= github.com/aws/aws-sdk-go-v2/credentials v1.4.0/go.mod h1:dgGR+Qq7Wjcd4AOAW5Rf5Tnv3+x7ed6kETXyS9WCuAY= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 h1:OxTAgH8Y4BXHD6PGCJ8DHx2kaZPCQfSTqmDsdRZFezE= +github.com/aws/aws-sdk-go-v2/credentials v1.16.16 h1:8q6Rliyv0aUFAVtzaldUEcS+T5gbadPbWdV1WcAddK8= +github.com/aws/aws-sdk-go-v2/credentials v1.16.16/go.mod h1:UHVZrdUsv63hPXFo1H7c5fEneoVo9UXiz36QG1GEPi0= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0/go.mod h1:CpNzHK9VEFUCknu50kkB8z58AH2B5DvPP7ea1LHve/Y= -github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2 h1:d95cddM3yTm4qffj3P6EnP+TzX1SSkWaQypXSgT/hpA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw= github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2/go.mod h1:BQV0agm+JEhqR+2RT5e1XTFIDcAAV0eW6z2trp+iduw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 h1:VNJ5NLBteVXEwE2F1zEXVmyIH58mZ6kIQGJoC7C+vkg= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 h1:n3GDfwqF2tzEkXlv5cuy4iy7LpKDtqDMcNLfZDu9rls= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0/go.mod h1:R1KK+vY8AfalhG1AOu5e35pOD2SdoPKQCFLTvnxiohk= -github.com/aws/aws-sdk-go-v2/service/kms v1.5.0 h1:10e9mzaaYIIePEuxUzW5YJ8LKHNG/NX63evcvS3ux9U= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino= github.com/aws/aws-sdk-go-v2/service/kms v1.5.0/go.mod h1:w7JuP9Oq1IKMFQPkNe3V6s9rOssXzOVEMNEqK1L1bao= +github.com/aws/aws-sdk-go-v2/service/kms v1.27.9 h1:W9PbZAZAEcelhhjb7KuwUtf+Lbc+i7ByYJRuWLlnxyQ= +github.com/aws/aws-sdk-go-v2/service/kms v1.27.9/go.mod h1:2tFmR7fQnOdQlM2ZCEPpFnBIQD1U8wmXmduBgZbOag0= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.6.0/go.mod h1:B+7C5UKdVq1ylkI/A6O8wcurFtaux0R1njePNPtKwoA= github.com/aws/aws-sdk-go-v2/service/ssm v1.10.0/go.mod h1:4dXS5YNqI3SNbetQ7X7vfsMlX6ZnboJA2dulBwJx7+g= -github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 h1:sHXMIKYS6YiLPzmKSvDpPmOpJDHxmAUgbiF49YNVztg= github.com/aws/aws-sdk-go-v2/service/sso v1.4.0/go.mod h1:+1fpWnL96DL23aXPpMGbsmKe8jLTEfbjuQoA4WS1VaA= -github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 h1:1at4e5P+lvHNl2nUktdM2/v+rpICg/QSEr9TO/uW9vU= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 h1:eajuO3nykDPdYicLlP3AGgOyVN3MOlFmZv7WGTuJPow= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.7/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 h1:QPMJf+Jw8E1l7zqhZmMlFw6w1NmfkfiSK8mS4zOx3BA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8= github.com/aws/aws-sdk-go-v2/service/sts v1.7.0/go.mod h1:0qcSMCyASQPN2sk/1KQLQ2Fh6yq8wm0HSDAimPhzCoM= -github.com/aws/smithy-go v1.8.0 h1:AEwwwXQZtUwP5Mz506FeXXrKBe0jA8gVM+1gEcSRooc= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= +github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= -github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= +github.com/beevik/etree v1.3.0 h1:hQTc+pylzIKDb23yYprodCWWTt+ojFfUZyzU09a/hmU= +github.com/beevik/etree v1.3.0/go.mod h1:aiPf89g/1k3AShMVAzriilpcE4R/Vuor90y83zVZWFc= github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -320,11 +346,9 @@ github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBS github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101 h1:7To3pQ+pZo0i3dsWEbinPNFs5gPSBOsJtx3wTT94VBY= +github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -333,11 +357,12 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/crewjam/saml v0.0.0-20190521120225-344d075952c9/go.mod h1:w5eu+HNtubx+kRpQL6QFT2F3yIFfYVe6+EzOFVU7Hko= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= @@ -412,8 +437,9 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/facebookincubator/flog v0.0.0-20190930132826-d2511d0ce33c h1:KqlxcP2nuOcMjudCvK0qME2K/aFBDH+xcvYv7HYQaYc= github.com/facebookincubator/flog v0.0.0-20190930132826-d2511d0ce33c/go.mod h1:QGzNH9ujQ2ZUr/CjDGZGWeDAVStrWNjHeEcjJL96Nuk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -421,8 +447,8 @@ github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= -github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= @@ -478,8 +504,8 @@ github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KE github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= @@ -517,12 +543,13 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -630,19 +657,20 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/rpmpack v0.0.0-20210518075352-dc539ef4f2ea h1:Fv9Ni1vIq9+Gv4Sm0Xq+NnPYcnsMbdNhJ4Cu4rkbPBM= github.com/google/rpmpack v0.0.0-20210518075352-dc539ef4f2ea/go.mod h1:+y9lKiqDhR4zkLl+V9h4q0rdyrYVsWWm6LLCQP33DIk= -github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= -github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8= github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= -github.com/googleapis/enterprise-certificate-proxy v0.2.4 h1:uGy6JWR/uMIILU8wbf+OkstIrNiMjGpEIyhx8f6W7s4= -github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -735,8 +763,9 @@ github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k= github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= @@ -778,8 +807,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/kolide/kit v0.0.0-20221107170827-fb85e3d59eab h1:KVR7cs+oPyy85i+8t1ZaNSy1bymCy5FuWyt51pdrXu4= github.com/kolide/kit v0.0.0-20221107170827-fb85e3d59eab/go.mod h1:OYYulo9tUqRadRLwB0+LE914sa1ui2yL7OrcU3Q/1XY= github.com/kolide/launcher v1.0.12 h1:f2uT1kKYGIbj/WVsHDc10f7MIiwu8MpmgwaGaT7D09k= @@ -832,8 +861,8 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -911,8 +940,8 @@ github.com/open-policy-agent/opa v0.44.0/go.mod h1:YpJaFIk5pq89n/k72c1lVvfvR5uop github.com/opencensus-integrations/ocsql v0.1.1/go.mod h1:ozPYpNVBHZsX33jfoQPO5TlI5lqh0/3R36kirEqJKAM= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec= -github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.1.0-rc6 h1:XDqvyKsJEbRtATzkgItUqBA7QHk58yxX1Ov9HERHNqU= +github.com/opencontainers/image-spec v1.1.0-rc6/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/oschwald/geoip2-golang v1.8.0 h1:KfjYB8ojCEn/QLqsDU0AzrJ3R5Qa9vFlx3z6SLNcKTs= github.com/oschwald/geoip2-golang v1.8.0/go.mod h1:R7bRvYjOeaoenAp9sKRS8GX5bJWcZ0laWO5+DauEktw= github.com/oschwald/maxminddb-golang v1.10.0 h1:Xp1u0ZhqkSuopaKmk1WwHtjF0H9Hd9181uj2MQ5Vndg= @@ -978,9 +1007,9 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs= -github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= +github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russellhaering/goxmldsig v0.0.0-20180430223755-7acd5e4a6ef7/go.mod h1:Oz4y6ImuOQZxynhbSXk7btjEfNBtGlj2dcaOvXl2FSM= github.com/russellhaering/goxmldsig v1.2.0 h1:Y6GTTc9Un5hCxSzVz4UIWQ/zuVwDvzJk80guqzwx6Vg= github.com/russellhaering/goxmldsig v1.2.0/go.mod h1:gM4MDENBQf7M+V824SGfyIUVFWydB7n0KkEubVJl+Tw= @@ -991,6 +1020,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/saferwall/pe v1.5.2 h1:h5lLtLsyxGHQ9dN6cd8EfeLEBEo5gdqJpkuw4o4vTMY= github.com/saferwall/pe v1.5.2/go.mod h1:SNzv3cdgk8SBI0UwHfyTcdjawfdnN+nbydnEL7GZ25s= +github.com/sassoftware/relic/v7 v7.6.2 h1:rS44Lbv9G9eXsukknS4mSjIAuuX+lMq/FnStgmZlUv4= +github.com/sassoftware/relic/v7 v7.6.2/go.mod h1:kjmP0IBVkJZ6gXeAu35/KCEfca//+PKM6vTAsyDPY+k= github.com/scjalliance/comshim v0.0.0-20230315213746-5e51f40bd3b9 h1:rc/CcqLH3lh8n+csdOuDfP+NuykE0U6AeYSJJHKDgSg= github.com/scjalliance/comshim v0.0.0-20230315213746-5e51f40bd3b9/go.mod h1:a/83NAfUXvEuLpmxDssAXxgUgrEy12MId3Wd7OTs76s= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -1042,8 +1073,8 @@ github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= -github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -1104,8 +1135,8 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= -github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= +github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli/v2 v2.23.5 h1:xbrU7tAYviSpqeR3X4nEFWUdB/uDZ6DE+HxmRU7Xtyw= github.com/urfave/cli/v2 v2.23.5/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/vartanbeno/go-reddit/v2 v2.0.0 h1:fxYMqx5lhbmJ3yYRN1nnQC/gecRB3xpUS2BbG7GLpsk= @@ -1143,6 +1174,8 @@ github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLE github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +go.einride.tech/aip v0.66.0 h1:XfV+NQX6L7EOYK11yoHHFtndeaWh3KbD9/cN/6iWEt8= +go.einride.tech/aip v0.66.0/go.mod h1:qAhMsfT7plxBX+Oy7Huol6YUvZ0ZzdUz26yZsQwfl1M= go.elastic.co/apm/module/apmgorilla/v2 v2.3.0 h1:jHw8N252UTwKTk945+Am8AaawhHC6DWpFVeTXQO8Gko= go.elastic.co/apm/module/apmgorilla/v2 v2.3.0/go.mod h1:2LXDBbVhFf9rF65jZecvl78IZMuvSRldQ+9A/fjfIo0= go.elastic.co/apm/module/apmhttp/v2 v2.3.0 h1:yGZyp26uJXUCfRTwvMmDt1d1jJrHgTBBncZfpYAxR8s= @@ -1173,21 +1206,25 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.44.0 h1:QaNUlLvmettd1vnmFHrgBYQHearxWP3uO4h4F3pVtkM= go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.44.0/go.mod h1:cJu+5jZwoZfkBOECSFtBZK/O7h/pY5djn0fwnIGnQ4A= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 h1:UNQQKPfTDe1J81ViolILjTKPr9WetKW6uei2hFgJmFs= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw= go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= -go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= -go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= +go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= -go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= -go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= +go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= -go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= -go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= +go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= -go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= -go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= +go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= @@ -1195,8 +1232,8 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= @@ -1222,14 +1259,14 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= @@ -1372,8 +1409,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1468,9 +1505,9 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1502,8 +1539,8 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1521,7 +1558,6 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1610,8 +1646,8 @@ google.golang.org/api v0.52.0/go.mod h1:Him/adpjt0sxtkWViy0b6xyKW/SD71CwdJ7HqJo7 google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.128.0 h1:RjPESny5CnQRn9V6siglged+DZCgfu9l6mO9dkX9VOg= -google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750= +google.golang.org/api v0.161.0 h1:oYzk/bs26WN10AV7iU7MVJVXBH8oCPS2hHyBiEeFoSU= +google.golang.org/api v0.161.0/go.mod h1:0mu0TpK33qnydLvWqbImq2b1eQ5FHRSDCBzAxX9ZHyw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1620,8 +1656,9 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1682,12 +1719,12 @@ google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEc google.golang.org/genproto v0.0.0-20210825212027-de86158e7fda/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0= -google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk= -google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a h1:myvhA4is3vrit1a6NZCWBIwN0kNEnX21DJOJX/NvIfI= -google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:SUBoKXbI1Efip18FClrQVGjWcyd0QZd8KkvdP34t7ww= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a h1:a2MQQVoTo96JC9PMGtGBymLp7+/RzpFc2yX/9WfFg1c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= +google.golang.org/genproto v0.0.0-20240205150955-31a09d347014 h1:g/4bk7P6TPMkAUbUhquq98xey1slwvuVJPosdBqYJlU= +google.golang.org/genproto v0.0.0-20240205150955-31a09d347014/go.mod h1:xEgQu1e4stdSSsxPDK8Azkrk/ECl5HvdPf6nbZrTS5M= +google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe h1:0poefMBYvYbs7g5UkjS6HcxBPaTRAmznle9jnxYoAI8= +google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe h1:bQnxqljG/wqi4NTXu2+DJ3n7APcEA882QZ1JvhQAq9o= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1713,9 +1750,8 @@ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= -google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0= +google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1783,12 +1819,13 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= -howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM= howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= +howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM= +howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -software.sslmate.com/src/go-pkcs12 v0.0.0-20210415151418-c5206de65a78 h1:SqYE5+A2qvRhErbsXFfUEUmpWEKxxRSMgGLkvRAFOV4= -software.sslmate.com/src/go-pkcs12 v0.0.0-20210415151418-c5206de65a78/go.mod h1:B7Wf0Ya4DHF9Yw+qfZuJijQYkWicqDa+79Ytmmq3Kjg= +software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k= +software.sslmate.com/src/go-pkcs12 v0.4.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI= diff --git a/pkg/file/deb.go b/pkg/file/deb.go index 07f51cf6c7..f224d8f3f9 100644 --- a/pkg/file/deb.go +++ b/pkg/file/deb.go @@ -6,6 +6,7 @@ import ( "bytes" "compress/bzip2" "compress/gzip" + "crypto/sha256" "errors" "fmt" "io" @@ -19,15 +20,17 @@ import ( // ExtractDebMetadata extracts the name and version metadata from a .deb file , // a debian installer package which is in archive format. -func ExtractDebMetadata(b []byte) (name, version string, err error) { - r := ar.NewReader(bytes.NewReader(b)) +func ExtractDebMetadata(r io.Reader) (name, version string, shaSum []byte, err error) { + h := sha256.New() + r = io.TeeReader(r, h) + rr := ar.NewReader(r) for { - hdr, err := r.Next() + hdr, err := rr.Next() if err == io.EOF { break } else if err != nil { - return "", "", fmt.Errorf("failed to advance to next file in archive: %w", err) + return "", "", nil, fmt.Errorf("failed to advance to next file in archive: %w", err) } name := path.Clean(hdr.Name) @@ -36,12 +39,26 @@ func ExtractDebMetadata(b []byte) (name, version string, err error) { if ext == ".tar" { ext = "" } - return parseControl(r, ext) + name, version, err = parseControl(rr, ext) + if err != nil { + return "", "", nil, err + } + + // ensure the whole file is read to get the correct hash + if _, err := io.Copy(io.Discard, r); err != nil { + return "", "", nil, fmt.Errorf("failed to read all content: %w", err) + } + return name, version, h.Sum(nil), nil } } + // ensure the whole file is read to get the correct hash + if _, err := io.Copy(io.Discard, r); err != nil { + return "", "", nil, fmt.Errorf("failed to read all content: %w", err) + } + // no control.tar file found, return empty information - return "", "", nil + return "", "", h.Sum(nil), nil } // parseControl adapted from diff --git a/pkg/file/file.go b/pkg/file/file.go index 23745468bb..833cc88739 100644 --- a/pkg/file/file.go +++ b/pkg/file/file.go @@ -12,18 +12,21 @@ import ( ) // ExtractInstallerMetadata extracts the software name and version from the -// installer file. The format of the installer is determined based on the -// extension of the filename. -func ExtractInstallerMetadata(filename string, b []byte) (name, version string, err error) { +// installer file and returns them along with the sha256 hash of the bytes. The +// format of the installer is determined based on the extension of the +// filename. +func ExtractInstallerMetadata(filename string, r io.Reader) (name, version string, shaSum []byte, err error) { switch ext := filepath.Ext(filename); ext { case ".deb": - return ExtractDebMetadata(b) + return ExtractDebMetadata(r) case ".exe": - return ExtractPEMetadata(b) + return ExtractPEMetadata(r) case ".pkg": - return ExtractXARMetadata(b) + return ExtractXARMetadata(r) + case ".msi": + return ExtractMSIMetadata(r) default: - return "", "", fmt.Errorf("unsupported file type: %s", ext) + return "", "", nil, fmt.Errorf("unsupported file type: %s", ext) } } diff --git a/pkg/file/file_test.go b/pkg/file/file_test.go index 3e0bd9c7b7..231031ccff 100644 --- a/pkg/file/file_test.go +++ b/pkg/file/file_test.go @@ -1,6 +1,7 @@ package file_test import ( + "encoding/hex" "io/fs" "os" "path/filepath" @@ -94,19 +95,19 @@ func TestExists(t *testing.T) { // TestExtractInstallerMetadata tests the ExtractInstallerMetadata function. It // calls the function for every file under testdata/installers and checks that -// it returns the expected metadata by comparing it to the software name and -// version in the filename. +// it returns the expected metadata by comparing it to the software name, +// version and hash in the filename. // // The filename should have the following format: // -// $[$]. +// $$[$]. // // That is, it breaks the file name at the dollar sign and the first part is -// the expected name, the second is the expected version. Note that by default, -// files in testdata/installers are NOT included in git, so the test files must -// be added manually (for size and licenses considerations). Why the dollar -// sign? Because dots, dashes and underlines are more likely to be part of the -// name or version. +// the expected name, the second is the expected version, the third is the +// hex-encoded hash. Note that by default, files in testdata/installers are NOT +// included in git, so the test files must be added manually (for size and +// licenses considerations). Why the dollar sign? Because dots, dashes and +// underlines are more likely to be part of the name or version. func TestExtractInstallerMetadata(t *testing.T) { dents, err := os.ReadDir(filepath.Join("testdata", "installers")) if err != nil { @@ -119,18 +120,20 @@ func TestExtractInstallerMetadata(t *testing.T) { } t.Run(dent.Name(), func(t *testing.T) { parts := strings.Split(strings.TrimSuffix(dent.Name(), filepath.Ext(dent.Name())), "$") - if len(parts) < 2 { - t.Fatalf("invalid filename, expected at least 2 sections, got %d: %s", len(parts), dent.Name()) + if len(parts) < 3 { + t.Fatalf("invalid filename, expected at least 3 sections, got %d: %s", len(parts), dent.Name()) } - wantName, wantVersion := parts[0], parts[1] + wantName, wantVersion, wantHash := parts[0], parts[1], parts[2] - content, err := os.ReadFile(filepath.Join("testdata", "installers", dent.Name())) + f, err := os.Open(filepath.Join("testdata", "installers", dent.Name())) require.NoError(t, err) + defer f.Close() - name, version, err := file.ExtractInstallerMetadata(dent.Name(), content) + name, version, hash, err := file.ExtractInstallerMetadata(dent.Name(), f) require.NoError(t, err) assert.Equal(t, wantName, name) assert.Equal(t, wantVersion, version) + assert.Equal(t, wantHash, hex.EncodeToString(hash)) }) } } diff --git a/pkg/file/msi.go b/pkg/file/msi.go new file mode 100644 index 0000000000..270e5242e4 --- /dev/null +++ b/pkg/file/msi.go @@ -0,0 +1,276 @@ +package file + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "io" + "strings" + + "github.com/sassoftware/relic/v7/lib/comdoc" +) + +func ExtractMSIMetadata(r io.Reader) (name, version string, shaSum []byte, err error) { + h := sha256.New() + r = io.TeeReader(r, h) + b, err := io.ReadAll(r) + if err != nil { + return "", "", nil, fmt.Errorf("failed to read all content: %w", err) + } + + rr := bytes.NewReader(b) + c, err := comdoc.ReadFile(rr) + if err != nil { + return "", "", nil, fmt.Errorf("reading msi file: %w", err) + } + defer c.Close() + + e, err := c.ListDir(nil) + if err != nil { + return "", "", nil, fmt.Errorf("listing files in msi: %w", err) + } + + // the product name and version are stored in the Property table, but the + // strings are interned in the _StringData table (which requires the + // _StringPool to decode). The structure of the tables is found in the + // _Columns table. + targetedTables := map[string]io.Reader{ + "Table._StringData": nil, + "Table._StringPool": nil, + "Table._Columns": nil, + "Table.Property": nil, + } + for _, ee := range e { + if ee.Type != comdoc.DirStream { + continue + } + + name := msiDecodeName(ee.Name()) + if _, ok := targetedTables[name]; ok { + rr, err := c.ReadStream(ee) + if err != nil { + return "", "", nil, fmt.Errorf("opening file stream %s: %w", name, err) + } + targetedTables[name] = rr + } + } + + // all tables must've been found + for k, v := range targetedTables { + if v == nil { + return "", "", nil, fmt.Errorf("table %s not found in the .msi", k) + } + } + + allStrings, err := decodeStrings(targetedTables["Table._StringData"], targetedTables["Table._StringPool"]) + if err != nil { + return "", "", nil, err + } + propTbl, err := decodePropertyTableColumns(targetedTables["Table._Columns"], allStrings) + if err != nil { + return "", "", nil, err + } + props, err := decodePropertyTable(targetedTables["Table.Property"], propTbl, allStrings) + if err != nil { + return "", "", nil, err + } + + return strings.TrimSpace(props["ProductName"]), strings.TrimSpace(props["ProductVersion"]), h.Sum(nil), nil +} + +type msiTable struct { + Name string + Cols []msiColumn +} + +type msiColumn struct { + Number int + Name string + Attributes uint16 +} + +func (c msiColumn) Type() msiType { + if c.Attributes&0x0F00 < 0x800 { + return msiType(c.Attributes & 0xFFF) + } + return msiType(c.Attributes & 0xF00) +} + +type msiType uint16 + +// column types +const ( + msiLong msiType = 0x104 + msiShort msiType = 0x502 + msiBinary msiType = 0x900 + msiString msiType = 0xD00 + msiStringLocalized msiType = 0xF00 + msiUnknown msiType = 0 +) + +func decodePropertyTable(propReader io.Reader, table *msiTable, strings []string) (map[string]string, error) { + // The Property table is a table of key-value pairs. Ensure the table has the + // expected format, otherwise we cannot extract the information. + if len(table.Cols) != 2 || table.Cols[0].Type() != msiString || table.Cols[1].Type() != msiStringLocalized { + return nil, errors.New("unexpected Property table structure") + } + + const propTableRowSize = 4 // 2 uint16s + + b, err := io.ReadAll(propReader) + if err != nil { + return nil, fmt.Errorf("failed to read columns table: %w", err) + } + rowCount := len(b) / propTableRowSize + propReader = bytes.NewReader(b) + + cols := [][]uint16{ + make([]uint16, 0, rowCount), + make([]uint16, 0, rowCount), + } + for i := 0; i < 2; i++ { + for j := 0; j < rowCount; j++ { + var v uint16 + err := binary.Read(propReader, binary.LittleEndian, &v) + if err != nil { + return nil, fmt.Errorf("failed to read column %d: %w", i, err) + } + cols[i] = append(cols[i], v) + } + } + + kv := make(map[string]string, rowCount) + for i := 0; i < rowCount; i++ { + kv[strings[cols[0][i]-1]] = strings[cols[1][i]-1] + } + return kv, nil +} + +func decodePropertyTableColumns(colReader io.Reader, strings []string) (*msiTable, error) { + const colTableRowSize = 8 // 4 uint16s + + // Columns table has 4 columns: + // - table name id (1-based index in strings array) + // - col number + // - col name id (1-based index in strings array) + // - col attributes (type) + // + // But to make things interesting, those are stored per column, so all first + // columns are stored for all rows, then all second columns for all rows, + // etc. + + b, err := io.ReadAll(colReader) + if err != nil { + return nil, fmt.Errorf("failed to read columns table: %w", err) + } + rowCount := len(b) / colTableRowSize + colReader = bytes.NewReader(b) + + cols := [][]uint16{ + make([]uint16, 0, rowCount), + make([]uint16, 0, rowCount), + make([]uint16, 0, rowCount), + make([]uint16, 0, rowCount), + } + for i := 0; i < 4; i++ { + for j := 0; j < rowCount; j++ { + var v uint16 + err := binary.Read(colReader, binary.LittleEndian, &v) + if err != nil { + return nil, fmt.Errorf("failed to read column %d: %w", i, err) + } + cols[i] = append(cols[i], v) + } + } + + var tbl msiTable + for i := 0; i < rowCount; i++ { + tblID, colNum, colNameID, colAttr := cols[0][i], cols[1][i], cols[2][i], cols[3][i] + + tableName := strings[tblID-1] + if tableName == "Property" { + tbl.Name = tableName + tbl.Cols = append(tbl.Cols, msiColumn{ + Number: int(colNum), + Name: strings[colNameID-1], + Attributes: colAttr, + }) + } + } + if tbl.Name == "" { + return nil, errors.New("Property table not found in columns table") + } + return &tbl, nil +} + +func decodeStrings(dataReader, poolReader io.Reader) ([]string, error) { + type header struct { + Codepage uint16 + Unknown uint16 + } + var poolHeader header + // pool data starts with 2 uint16 for the codepage and an unknown value + err := binary.Read(poolReader, binary.LittleEndian, &poolHeader) + if err != nil { + if err == io.EOF { + return nil, io.ErrUnexpectedEOF + } + return nil, fmt.Errorf("failed to read pool header: %w", err) + } + + type entry struct { + Size uint16 + RefCount uint16 + } + var stringEntry entry + var stringTable []string + for { + err := binary.Read(poolReader, binary.LittleEndian, &stringEntry) + if err != nil { + if err == io.EOF { + break + } + return nil, fmt.Errorf("failed to read pool entry: %w", err) + } + buf := make([]byte, stringEntry.Size) + if _, err := io.ReadFull(dataReader, buf); err != nil { + return nil, fmt.Errorf("failed to read string data: %w", err) + } + stringTable = append(stringTable, string(buf)) + } + return stringTable, nil +} + +func msiDecodeName(msiName string) string { + out := "" + for _, x := range msiName { + if x >= 0x3800 && x < 0x4800 { + x -= 0x3800 + out += string(msiDecodeRune(x&0x3f)) + string(msiDecodeRune(x>>6)) + } else if x >= 0x4800 && x < 0x4840 { + x -= 0x4800 + out += string(msiDecodeRune(x)) + } else if x == 0x4840 { + out += "Table." + } else { + out += string(x) + } + } + return out +} + +func msiDecodeRune(x rune) rune { + if x < 10 { + return x + '0' + } else if x < 10+26 { + return x - 10 + 'A' + } else if x < 10+26+26 { + return x - 10 - 26 + 'a' + } else if x == 10+26+26 { + return '.' + } else { + return '_' + } +} diff --git a/pkg/file/pe.go b/pkg/file/pe.go index 3f6220d264..8b09319bf7 100644 --- a/pkg/file/pe.go +++ b/pkg/file/pe.go @@ -1,7 +1,9 @@ package file import ( + "crypto/sha256" "fmt" + "io" "strings" "github.com/saferwall/pe" @@ -9,7 +11,14 @@ import ( // ExtractPEMetadata extracts the name and version metadata from a .exe file in // the Portable Executable (PE) format. -func ExtractPEMetadata(b []byte) (name, version string, err error) { +func ExtractPEMetadata(r io.Reader) (name, version string, shaSum []byte, err error) { + h := sha256.New() + r = io.TeeReader(r, h) + b, err := io.ReadAll(r) + if err != nil { + return "", "", nil, fmt.Errorf("failed to read all content: %w", err) + } + // cannot use the "Fast" option, we need the data directories for the // resources to be available. pep, err := pe.NewBytes(b, &pe.Options{ @@ -29,17 +38,17 @@ func ExtractPEMetadata(b []byte) (name, version string, err error) { OmitCLRHeaderDirectory: true, }) if err != nil { - return "", "", fmt.Errorf("error creating PE file: %w", err) + return "", "", nil, fmt.Errorf("error creating PE file: %w", err) } defer pep.Close() if err := pep.Parse(); err != nil { - return "", "", fmt.Errorf("error parsing PE file: %w", err) + return "", "", nil, fmt.Errorf("error parsing PE file: %w", err) } v, err := pep.ParseVersionResources() if err != nil { - return "", "", fmt.Errorf("error parsing PE version resources: %w", err) + return "", "", nil, fmt.Errorf("error parsing PE version resources: %w", err) } - return strings.TrimSpace(v["ProductName"]), strings.TrimSpace(v["ProductVersion"]), nil + return strings.TrimSpace(v["ProductName"]), strings.TrimSpace(v["ProductVersion"]), h.Sum(nil), nil } diff --git a/pkg/file/xar.go b/pkg/file/xar.go index 0bbe49d278..3ec779345c 100644 --- a/pkg/file/xar.go +++ b/pkg/file/xar.go @@ -22,6 +22,7 @@ import ( "bytes" "compress/zlib" "crypto" + "crypto/sha256" "encoding/binary" "encoding/xml" "errors" @@ -101,7 +102,12 @@ type xmlFile struct { } type distributionXML struct { - PkgRef []pkgRef `xml:"pkg-ref"` + Title string `xml:"title"` + PkgRef []pkgRef `xml:"pkg-ref"` + Product struct { + ID string `xml:"id,attr"` + Version string `xml:"version,attr"` + } `xml:"product"` } type pkgRef struct { @@ -113,17 +119,24 @@ type pkgRef struct { // ExtractXARMetadata extracts the name and version metadata from a .pkg file // in the XAR format. -func ExtractXARMetadata(b []byte) (name, version string, err error) { +func ExtractXARMetadata(r io.Reader) (name, version string, shaSum []byte, err error) { var hdr xarHeader - r := bytes.NewReader(b) - if err := binary.Read(r, binary.BigEndian, &hdr); err != nil { - return "", "", fmt.Errorf("decode xar header: %w", err) + h := sha256.New() + r = io.TeeReader(r, h) + b, err := io.ReadAll(r) + if err != nil { + return "", "", nil, fmt.Errorf("failed to read all content: %w", err) } - zr, err := zlib.NewReader(io.LimitReader(r, hdr.CompressedSize)) + rr := bytes.NewReader(b) + if err := binary.Read(rr, binary.BigEndian, &hdr); err != nil { + return "", "", nil, fmt.Errorf("decode xar header: %w", err) + } + + zr, err := zlib.NewReader(io.LimitReader(rr, hdr.CompressedSize)) if err != nil { - return "", "", fmt.Errorf("create zlib reader: %w", err) + return "", "", nil, fmt.Errorf("create zlib reader: %w", err) } defer zr.Close() @@ -131,14 +144,14 @@ func ExtractXARMetadata(b []byte) (name, version string, err error) { decoder := xml.NewDecoder(zr) decoder.Strict = false if err := decoder.Decode(&root); err != nil { - return "", "", fmt.Errorf("decode xar xml: %w", err) + return "", "", nil, fmt.Errorf("decode xar xml: %w", err) } heapOffset := xarHeaderSize + hdr.CompressedSize for _, f := range root.TOC.Files { if f.Name == "Distribution" { var fileReader io.Reader - heapReader := io.NewSectionReader(r, heapOffset, int64(len(b))-heapOffset) + heapReader := io.NewSectionReader(rr, heapOffset, int64(len(b))-heapOffset) fileReader = io.NewSectionReader(heapReader, f.Data.Offset, f.Data.Length) // the distribution file can be compressed differently than the TOC, the @@ -148,7 +161,7 @@ func ExtractXARMetadata(b []byte) (name, version string, err error) { // (invalid header), but it works with zlib. zr, err := zlib.NewReader(fileReader) if err != nil { - return "", "", fmt.Errorf("create zlib reader: %w", err) + return "", "", nil, fmt.Errorf("create zlib reader: %w", err) } defer zr.Close() fileReader = zr @@ -158,22 +171,41 @@ func ExtractXARMetadata(b []byte) (name, version string, err error) { contents, err := io.ReadAll(fileReader) if err != nil { - return "", "", fmt.Errorf("reading Distribution file: %w", err) + return "", "", nil, fmt.Errorf("reading Distribution file: %w", err) } var distXML distributionXML if err := xml.Unmarshal(contents, &distXML); err != nil { - return "", "", fmt.Errorf("unmarshal Distribution XML: %w", err) + return "", "", nil, fmt.Errorf("unmarshal Distribution XML: %w", err) } + // Get the name from (in order of priority): + // - Title + // - product.id + // - pkg-ref[0].id + // + // Get the version from (in order of priority): + // - product.version + // - pkg-ref[0].version + name := strings.TrimSpace(distXML.Title) + if name == "" { + name = strings.TrimSpace(distXML.Product.ID) + } + version := strings.TrimSpace(distXML.Product.Version) if len(distXML.PkgRef) > 0 { - return strings.TrimSpace(distXML.PkgRef[0].ID), strings.TrimSpace(distXML.PkgRef[0].Version), nil + if name == "" { + name = strings.TrimSpace(distXML.PkgRef[0].ID) + } + if version == "" { + version = strings.TrimSpace(distXML.PkgRef[0].Version) + } + return name, version, h.Sum(nil), nil } break } } - return "", "", nil + return "", "", h.Sum(nil), nil } // CheckPKGSignature checks if the provided bytes correspond to a signed pkg From 92e540aee57ef8519bc8a95e30d59eea9da0aef0 Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Wed, 1 May 2024 14:15:59 -0300 Subject: [PATCH 09/56] add scripts to add/remove software (#18649) for: - https://github.com/fleetdm/fleet/issues/18314 - https://github.com/fleetdm/fleet/issues/18315 - https://github.com/fleetdm/fleet/issues/18317 - https://github.com/fleetdm/fleet/issues/18316 # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Added/updated tests - [x] Manual QA for all new/changed functionality --- frontend/interfaces/software.ts | 4 + .../utilities/software_install_scripts.ts | 50 +++++++++++ pkg/file/management.go | 85 +++++++++++++++++++ pkg/file/management_test.go | 73 ++++++++++++++++ pkg/file/scripts/README.md | 15 ++++ pkg/file/scripts/install_deb.sh | 1 + pkg/file/scripts/install_exe.ps1 | 21 +++++ pkg/file/scripts/install_msi.ps1 | 9 ++ pkg/file/scripts/install_pkg.sh | 1 + pkg/file/scripts/remove_deb.sh | 1 + pkg/file/scripts/remove_exe.ps1 | 14 +++ pkg/file/scripts/remove_msi.ps1 | 9 ++ pkg/file/scripts/remove_pkg.sh | 8 ++ .../testdata/scripts/install_deb.sh.golden | 1 + .../testdata/scripts/install_exe.ps1.golden | 21 +++++ .../testdata/scripts/install_msi.ps1.golden | 9 ++ .../testdata/scripts/install_pkg.sh.golden | 1 + .../testdata/scripts/remove_deb.sh.golden | 1 + .../testdata/scripts/remove_exe.ps1.golden | 14 +++ .../testdata/scripts/remove_msi.ps1.golden | 9 ++ .../testdata/scripts/remove_pkg.sh.golden | 8 ++ webpack.config.js | 5 +- 22 files changed, 359 insertions(+), 1 deletion(-) create mode 100644 frontend/utilities/software_install_scripts.ts create mode 100644 pkg/file/management.go create mode 100644 pkg/file/management_test.go create mode 100644 pkg/file/scripts/README.md create mode 100644 pkg/file/scripts/install_deb.sh create mode 100644 pkg/file/scripts/install_exe.ps1 create mode 100644 pkg/file/scripts/install_msi.ps1 create mode 100644 pkg/file/scripts/install_pkg.sh create mode 100644 pkg/file/scripts/remove_deb.sh create mode 100644 pkg/file/scripts/remove_exe.ps1 create mode 100644 pkg/file/scripts/remove_msi.ps1 create mode 100644 pkg/file/scripts/remove_pkg.sh create mode 100644 pkg/file/testdata/scripts/install_deb.sh.golden create mode 100644 pkg/file/testdata/scripts/install_exe.ps1.golden create mode 100644 pkg/file/testdata/scripts/install_msi.ps1.golden create mode 100644 pkg/file/testdata/scripts/install_pkg.sh.golden create mode 100644 pkg/file/testdata/scripts/remove_deb.sh.golden create mode 100644 pkg/file/testdata/scripts/remove_exe.ps1.golden create mode 100644 pkg/file/testdata/scripts/remove_msi.ps1.golden create mode 100644 pkg/file/testdata/scripts/remove_pkg.sh.golden diff --git a/frontend/interfaces/software.ts b/frontend/interfaces/software.ts index 3ce5f5d268..8f47761baf 100644 --- a/frontend/interfaces/software.ts +++ b/frontend/interfaces/software.ts @@ -133,3 +133,7 @@ export const formatSoftwareType = ({ } return type; }; + +// ISoftwareInstallerType defines the supported installer types for +// software uploaded by the IT admin. +export type ISoftwareInstallerType = "pkg" | "msi" | "deb" | "exe"; diff --git a/frontend/utilities/software_install_scripts.ts b/frontend/utilities/software_install_scripts.ts new file mode 100644 index 0000000000..a26fcb0a15 --- /dev/null +++ b/frontend/utilities/software_install_scripts.ts @@ -0,0 +1,50 @@ +import { ISoftwareInstallerType } from "interfaces/software"; + +// @ts-ignore +import installPkg from "../../pkg/file/scripts/install_pkg.sh"; +// @ts-ignore +import installMsi from "../../pkg/file/scripts/install_msi.ps1"; +// @ts-ignore +import installExe from "../../pkg/file/scripts/install_exe.ps1"; +// @ts-ignore +import installDeb from "../../pkg/file/scripts/install_deb.sh"; + +const replaceVariables = (rawScript: string, installerPath: string): string => { + return rawScript.replace("$INSTALLER_PATH", installerPath); +}; + +/* + * getInstallScript returns a string with a script to install the + * provided software. + * + * Note that we don't do any sanitization of the arguments here, + * delegating that to the caller which should have the right context + * about what should be escaped. + * */ +const getInstallScript = ( + filetype: ISoftwareInstallerType, + path: string +): string => { + let rawScript: string; + switch (filetype) { + case "pkg": + rawScript = installPkg; + break; + case "msi": + rawScript = installMsi; + break; + case "deb": + rawScript = installDeb; + break; + case "exe": + rawScript = installExe; + break; + default: + // this should never happen as this function is type-guarded + throw new Error(`unsupported file type: ${filetype}`); + } + + return replaceVariables(rawScript, path); +}; + +export default getInstallScript; diff --git a/pkg/file/management.go b/pkg/file/management.go new file mode 100644 index 0000000000..28d2e7710a --- /dev/null +++ b/pkg/file/management.go @@ -0,0 +1,85 @@ +package file + +import ( + _ "embed" + "strings" +) + +type InstallerType string + +const ( + InstallerTypeMsi InstallerType = "msi" + InstallerTypeDeb InstallerType = "deb" + InstallerTypePkg InstallerType = "pkg" + InstallerTypeExe InstallerType = "exe" +) + +//go:embed scripts/install_pkg.sh +var installPkgScript string + +//go:embed scripts/install_msi.ps1 +var installMsiScript string + +//go:embed scripts/install_exe.ps1 +var installExeScript string + +//go:embed scripts/install_deb.sh +var installDebScript string + +// GetInstallScript returns a script that can be used to install the given +// installer based on the provided type +func GetInstallScript(installerType InstallerType, installerPath string) string { + var rawScript string + + switch installerType { + case InstallerTypeMsi: + rawScript = installMsiScript + case InstallerTypeDeb: + rawScript = installDebScript + case InstallerTypePkg: + rawScript = installPkgScript + case InstallerTypeExe: + rawScript = installExeScript + default: + return "" + } + + return replaceVars(rawScript, installerPath) +} + +//go:embed scripts/remove_exe.ps1 +var removeExeScript string + +//go:embed scripts/remove_pkg.sh +var removePkgScript string + +//go:embed scripts/remove_msi.ps1 +var removeMsiScript string + +//go:embed scripts/remove_deb.sh +var removeDebScript string + +// GetRemoveScript returns a script that can be used to remove the given +// installer based on the provided type +func GetRemoveScript(installerType InstallerType, installerPath string) string { + var rawScript string + + switch installerType { + case InstallerTypeMsi: + rawScript = removeMsiScript + case InstallerTypeDeb: + rawScript = removeDebScript + case InstallerTypePkg: + rawScript = removePkgScript + case InstallerTypeExe: + rawScript = removeExeScript + default: + return "" + } + + return replaceVars(rawScript, installerPath) +} + +func replaceVars(rawScript string, installerPath string) string { + return strings.Replace(rawScript, "$INSTALLER_PATH", installerPath, -1) +} diff --git a/pkg/file/management_test.go b/pkg/file/management_test.go new file mode 100644 index 0000000000..1b6af00f56 --- /dev/null +++ b/pkg/file/management_test.go @@ -0,0 +1,73 @@ +package file + +import ( + "flag" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +var ( + update = flag.Bool("update", false, "update the golden files of this test") +) + +func TestMain(m *testing.M) { + flag.Parse() + os.Exit(m.Run()) +} + +// Note: to update the goldens, run the tests with `-update`: +// +// go test ./pkg/file/... -update +func TestGetInstallAndRemoveScript(t *testing.T) { + scriptsByType := map[InstallerType][2]string{ + InstallerTypeMsi: { + "./scripts/install_msi.ps1", + "./scripts/remove_msi.ps1", + }, + InstallerTypePkg: { + "./scripts/install_pkg.sh", + "./scripts/remove_pkg.sh", + }, + InstallerTypeDeb: { + "./scripts/install_deb.sh", + "./scripts/remove_deb.sh", + }, + InstallerTypeExe: { + "./scripts/install_exe.ps1", + "./scripts/remove_exe.ps1", + }, + } + + for itype, scripts := range scriptsByType { + installerPath := "./foo/bar baz.f" + + gotScript := GetInstallScript(itype, installerPath) + assertGoldenMatches(t, scripts[0], gotScript, *update) + + gotScript = GetRemoveScript(itype, installerPath) + assertGoldenMatches(t, scripts[1], gotScript, *update) + } +} + +func assertGoldenMatches(t *testing.T, goldenFile string, actual string, update bool) { + t.Helper() + goldenPath := filepath.Join("testdata", goldenFile+".golden") + + f, err := os.OpenFile(goldenPath, os.O_RDWR|os.O_CREATE, 0644) + require.NoError(t, err) + defer f.Close() + + if update { + _, err := f.WriteString(actual) + require.NoError(t, err) + return + } + + content, err := io.ReadAll(f) + require.NoError(t, err) + require.Equal(t, string(content), actual) +} diff --git a/pkg/file/scripts/README.md b/pkg/file/scripts/README.md new file mode 100644 index 0000000000..42c81fe507 --- /dev/null +++ b/pkg/file/scripts/README.md @@ -0,0 +1,15 @@ +### File scripts + +This folder contains scripts to install/remove software for different types of installers. + +Scripts are stored on their own files for two reasons: + +1. Some of them are read and displayed in the UI. +2. It's helpful to have good syntax highlighting and easy ways to run them. + +#### Variables + +Because the scripts are shared between Go and JS, the convention is to declare variables using `$VAR_NAME` and document its intended usage here. + +- `$INSTALLER_PATH` path to the installer file. + diff --git a/pkg/file/scripts/install_deb.sh b/pkg/file/scripts/install_deb.sh new file mode 100644 index 0000000000..16d246663e --- /dev/null +++ b/pkg/file/scripts/install_deb.sh @@ -0,0 +1 @@ +apt-get install -f "$INSTALLER_PATH" diff --git a/pkg/file/scripts/install_exe.ps1 b/pkg/file/scripts/install_exe.ps1 new file mode 100644 index 0000000000..4aa91f5b1c --- /dev/null +++ b/pkg/file/scripts/install_exe.ps1 @@ -0,0 +1,21 @@ +$exeFilePath = "$INSTALLER_PATH" + +# extract the name of the executable to use as the sub-directory name +$exeName = [System.IO.Path]::GetFileName($exeFilePath) +$subDir = [System.IO.Path]::GetFileNameWithoutExtension($exeFilePath) + +# Program Files is the recommended location for any third-party software on Windows. +# +# Note: a x86 binary on a x64 system is supposed to go in +# $env:ProgramFiles(x86) but I didn't find a reliable way to get this +# information from the exe file. +$destinationPath = Join-Path -Path $env:ProgramFiles -ChildPath $subDir + +# check if the directory does not exist, and create it if necessary +if (-not (Test-Path -Path $destinationPath)) { + New-Item -ItemType Directory -Path $destinationPath +} + +# copy the .exe file to the new sub-directory +$destinationExePath = Join-Path -Path $destinationPath -ChildPath $exeName +Copy-Item -Path $exeFilePath -Destination $destinationExePath diff --git a/pkg/file/scripts/install_msi.ps1 b/pkg/file/scripts/install_msi.ps1 new file mode 100644 index 0000000000..e4f8d8ca90 --- /dev/null +++ b/pkg/file/scripts/install_msi.ps1 @@ -0,0 +1,9 @@ +$logFile = "${env:TEMP}/fleet-install-software.log" + +$installProcess = Start-Process msiexec.exe ` + -ArgumentList "/quiet /norestart /lv ${logFile} /i `"$INSTALLER_PATH`"" ` + -PassThru -Verb RunAs -Wait + +Get-Content $logFile -Tail 500 + +exit $instalProcess.ExitCode diff --git a/pkg/file/scripts/install_pkg.sh b/pkg/file/scripts/install_pkg.sh new file mode 100644 index 0000000000..783d2941a7 --- /dev/null +++ b/pkg/file/scripts/install_pkg.sh @@ -0,0 +1 @@ +installer -pkg "$INSTALLER_PATH" -target / diff --git a/pkg/file/scripts/remove_deb.sh b/pkg/file/scripts/remove_deb.sh new file mode 100644 index 0000000000..5de7123909 --- /dev/null +++ b/pkg/file/scripts/remove_deb.sh @@ -0,0 +1 @@ +apt-get remove -y $(dpkg -f "$INSTALLER_PATH" Package) diff --git a/pkg/file/scripts/remove_exe.ps1 b/pkg/file/scripts/remove_exe.ps1 new file mode 100644 index 0000000000..eae826c960 --- /dev/null +++ b/pkg/file/scripts/remove_exe.ps1 @@ -0,0 +1,14 @@ +$exeFilePath = "$INSTALLER_PATH" + +# extract the name of the executable to use as the sub-directory name +$exeName = [System.IO.Path]::GetFileName($exeFilePath) +$subDir = [System.IO.Path]::GetFileNameWithoutExtension($exeFilePath) + +# determine the correct Program Files directory based on OS architecture +$destinationPath = Join-Path -Path $env:ProgramFiles -ChildPath $subDir +$destinationExePath = Join-Path -Path $destinationPath -ChildPath $exeName + +# remove only the exe file, while at runtime other files could have been +# created in this folder, this is a naive approach to prevent forcing us to +# remove important folders by crafting a malicious file name. +Remove-Item -Path $destinationExePath diff --git a/pkg/file/scripts/remove_msi.ps1 b/pkg/file/scripts/remove_msi.ps1 new file mode 100644 index 0000000000..899a4c9bec --- /dev/null +++ b/pkg/file/scripts/remove_msi.ps1 @@ -0,0 +1,9 @@ +$logFile = "${env:TEMP}/fleet-remove-software.log" + +$removeProcess = Start-Process msiexec.exe ` + -ArgumentList "/quiet /norestart /lv ${logFile} /x `"$INSTALLER_PATH`"" ` + -PassThru -Verb RunAs -Wait + +Get-Content $logFile -Tail 500 + +exit $instalProcess.ExitCode diff --git a/pkg/file/scripts/remove_pkg.sh b/pkg/file/scripts/remove_pkg.sh new file mode 100644 index 0000000000..84ceb7a3fc --- /dev/null +++ b/pkg/file/scripts/remove_pkg.sh @@ -0,0 +1,8 @@ +# grab the identifier from the first PackageInfo we find. Those are placed in different locations depending on the installer +pkg_id=$(tar xOvf "$INSTALLER_PATH" --include='*PackageInfo*' 2>/dev/null | sed -n 's/.*identifier="\([^"]*\)".*/\1/p') + +# remove all the files and empty directories that were installed +pkgutil --files $pkg_id | tr '\n' '\0' | xargs -n 1 -0 rm -d + +# remove the receipt +pkgutil --forget $pkg_id diff --git a/pkg/file/testdata/scripts/install_deb.sh.golden b/pkg/file/testdata/scripts/install_deb.sh.golden new file mode 100644 index 0000000000..d0c8a10f6b --- /dev/null +++ b/pkg/file/testdata/scripts/install_deb.sh.golden @@ -0,0 +1 @@ +apt-get install -f "./foo/bar baz.f" diff --git a/pkg/file/testdata/scripts/install_exe.ps1.golden b/pkg/file/testdata/scripts/install_exe.ps1.golden new file mode 100644 index 0000000000..a3fa160bcc --- /dev/null +++ b/pkg/file/testdata/scripts/install_exe.ps1.golden @@ -0,0 +1,21 @@ +$exeFilePath = "./foo/bar baz.f" + +# extract the name of the executable to use as the sub-directory name +$exeName = [System.IO.Path]::GetFileName($exeFilePath) +$subDir = [System.IO.Path]::GetFileNameWithoutExtension($exeFilePath) + +# Program Files is the recommended location for any third-party software on Windows. +# +# Note: a x86 binary on a x64 system is supposed to go in +# $env:ProgramFiles(x86) but I didn't find a reliable way to get this +# information from the exe file. +$destinationPath = Join-Path -Path $env:ProgramFiles -ChildPath $subDir + +# check if the directory does not exist, and create it if necessary +if (-not (Test-Path -Path $destinationPath)) { + New-Item -ItemType Directory -Path $destinationPath +} + +# copy the .exe file to the new sub-directory +$destinationExePath = Join-Path -Path $destinationPath -ChildPath $exeName +Copy-Item -Path $exeFilePath -Destination $destinationExePath diff --git a/pkg/file/testdata/scripts/install_msi.ps1.golden b/pkg/file/testdata/scripts/install_msi.ps1.golden new file mode 100644 index 0000000000..4ffd31f6f9 --- /dev/null +++ b/pkg/file/testdata/scripts/install_msi.ps1.golden @@ -0,0 +1,9 @@ +$logFile = "${env:TEMP}/fleet-install-software.log" + +$installProcess = Start-Process msiexec.exe ` + -ArgumentList "/quiet /norestart /lv ${logFile} /i `"./foo/bar baz.f`"" ` + -PassThru -Verb RunAs -Wait + +Get-Content $logFile -Tail 500 + +exit $instalProcess.ExitCode diff --git a/pkg/file/testdata/scripts/install_pkg.sh.golden b/pkg/file/testdata/scripts/install_pkg.sh.golden new file mode 100644 index 0000000000..b736f14690 --- /dev/null +++ b/pkg/file/testdata/scripts/install_pkg.sh.golden @@ -0,0 +1 @@ +installer -pkg "./foo/bar baz.f" -target / diff --git a/pkg/file/testdata/scripts/remove_deb.sh.golden b/pkg/file/testdata/scripts/remove_deb.sh.golden new file mode 100644 index 0000000000..cdc206a19b --- /dev/null +++ b/pkg/file/testdata/scripts/remove_deb.sh.golden @@ -0,0 +1 @@ +apt-get remove -y $(dpkg -f "./foo/bar baz.f" Package) diff --git a/pkg/file/testdata/scripts/remove_exe.ps1.golden b/pkg/file/testdata/scripts/remove_exe.ps1.golden new file mode 100644 index 0000000000..0f2548bd6a --- /dev/null +++ b/pkg/file/testdata/scripts/remove_exe.ps1.golden @@ -0,0 +1,14 @@ +$exeFilePath = "./foo/bar baz.f" + +# extract the name of the executable to use as the sub-directory name +$exeName = [System.IO.Path]::GetFileName($exeFilePath) +$subDir = [System.IO.Path]::GetFileNameWithoutExtension($exeFilePath) + +# determine the correct Program Files directory based on OS architecture +$destinationPath = Join-Path -Path $env:ProgramFiles -ChildPath $subDir +$destinationExePath = Join-Path -Path $destinationPath -ChildPath $exeName + +# remove only the exe file, while at runtime other files could have been +# created in this folder, this is a naive approach to prevent forcing us to +# remove important folders by crafting a malicious file name. +Remove-Item -Path $destinationExePath diff --git a/pkg/file/testdata/scripts/remove_msi.ps1.golden b/pkg/file/testdata/scripts/remove_msi.ps1.golden new file mode 100644 index 0000000000..7ae7d04ec0 --- /dev/null +++ b/pkg/file/testdata/scripts/remove_msi.ps1.golden @@ -0,0 +1,9 @@ +$logFile = "${env:TEMP}/fleet-remove-software.log" + +$removeProcess = Start-Process msiexec.exe ` + -ArgumentList "/quiet /norestart /lv ${logFile} /x `"./foo/bar baz.f`"" ` + -PassThru -Verb RunAs -Wait + +Get-Content $logFile -Tail 500 + +exit $instalProcess.ExitCode diff --git a/pkg/file/testdata/scripts/remove_pkg.sh.golden b/pkg/file/testdata/scripts/remove_pkg.sh.golden new file mode 100644 index 0000000000..73d2e32cb1 --- /dev/null +++ b/pkg/file/testdata/scripts/remove_pkg.sh.golden @@ -0,0 +1,8 @@ +# grab the identifier from the first PackageInfo we find. Those are placed in different locations depending on the installer +pkg_id=$(tar xOvf "./foo/bar baz.f" --include='*PackageInfo*' 2>/dev/null | sed -n 's/.*identifier="\([^"]*\)".*/\1/p') + +# remove all the files and empty directories that were installed +pkgutil --files $pkg_id | tr '\n' '\0' | xargs -n 1 -0 rm -d + +# remove the receipt +pkgutil --forget $pkg_id diff --git a/webpack.config.js b/webpack.config.js index 1bd6aca7e1..8b06782811 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -71,7 +71,10 @@ const config = { filename: "[name]@[hash][ext]", }, }, - + { + test: /\.(sh|ps1)$/, + type: "asset/source", + }, { test: /(\.tsx?|\.jsx?)$/, exclude: /node_modules/, From ad11f075c1625812c456f15d81bace191c314557 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 1 May 2024 14:37:52 -0400 Subject: [PATCH 10/56] Add API endpoint to list host/device software (#18676) --- changes/18319-api-to-list-host-software | 1 + ...240424124712_AddSoftwareInstallerTables.go | 6 +- server/datastore/mysql/schema.sql | 2 +- server/datastore/mysql/software.go | 304 ++++++++++++ server/datastore/mysql/software_test.go | 436 ++++++++++++++++++ server/fleet/datastore.go | 2 + server/fleet/service.go | 4 + server/fleet/software_installer.go | 36 ++ server/mock/datastore_mock.go | 12 + server/service/devices.go | 39 ++ server/service/handler.go | 4 + server/service/hosts.go | 69 +++ server/service/hosts_test.go | 9 + server/service/integration_enterprise_test.go | 49 ++ 14 files changed, 969 insertions(+), 4 deletions(-) create mode 100644 changes/18319-api-to-list-host-software diff --git a/changes/18319-api-to-list-host-software b/changes/18319-api-to-list-host-software new file mode 100644 index 0000000000..feaf6422d1 --- /dev/null +++ b/changes/18319-api-to-list-host-software @@ -0,0 +1 @@ +* Added the `GET /api/v1/fleet/hosts/{id}/software` (and corresponding token-authenticated endpoint for the "My device" page) to list the installed (and available for install) software for the host. diff --git a/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go b/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go index 7e7dc7e15f..6c09e213b0 100644 --- a/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go +++ b/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS software_installers ( install_script_content_id int(10) unsigned NOT NULL, -- FK to the script_contents for the post-script uploaded by the IT admin to - -- be run after the software is installed + -- be run after the software is installed post_install_script_content_id int(10) unsigned DEFAULT NULL, -- used to track the ID retrieved from the storage containing the installer bytes @@ -70,7 +70,7 @@ CREATE TABLE IF NOT EXISTS software_installers ( REFERENCES teams (id) ON DELETE CASCADE ON UPDATE CASCADE, - + UNIQUE KEY idx_software_installers_team_id_title_id (global_or_team_id, title_id) ) @@ -120,7 +120,7 @@ CREATE TABLE IF NOT EXISTS host_software_installs ( REFERENCES software_installers (id) ON DELETE CASCADE ON UPDATE CASCADE, - UNIQUE KEY idx_host_software_installs_host_installer (host_id, software_installer_id), + KEY idx_host_software_installs_host_installer (host_id, software_installer_id), -- this index can be used to lookup results for a specific -- execution (execution ids, e.g. when updating the row for results) diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 861cc47d6e..9366e8d653 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -499,9 +499,9 @@ CREATE TABLE `host_software_installs` ( `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), - UNIQUE KEY `idx_host_software_installs_host_installer` (`host_id`,`software_installer_id`), UNIQUE KEY `idx_host_software_installs_execution_id` (`execution_id`), KEY `fk_host_software_installs_installer_id` (`software_installer_id`), + KEY `idx_host_software_installs_host_installer` (`host_id`,`software_installer_id`), CONSTRAINT `fk_host_software_installs_installer_id` FOREIGN KEY (`software_installer_id`) REFERENCES `software_installers` (`id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index e7ad231d28..532e473c6d 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -1721,3 +1721,307 @@ func (ds *Datastore) ListCVEs(ctx context.Context, maxAge time.Duration) ([]flee return result, nil } + +func (ds *Datastore) ListHostSoftware(ctx context.Context, hostID uint, includeAvailableForInstall bool, opts fleet.ListOptions) ([]*fleet.HostSoftwareWithInstaller, *fleet.PaginationMetadata, error) { + // `status` computed column assumes that all results (pre, install and post) + // are stored at once, so that if there is an exit code for the install + // script and none for the post-install, it is because there is no + // post-install. + const stmtInstalled = ` + SELECT + st.id, + st.name, + st.source, + si.filename as package_available_for_install, + hsi.created_at as last_install_installed_at, + hsi.execution_id as last_install_install_uuid, + CASE + WHEN hsi.post_install_script_exit_code IS NOT NULL AND + hsi.post_install_script_exit_code = 0 THEN ? -- installed + + WHEN hsi.post_install_script_exit_code IS NOT NULL AND + hsi.post_install_script_exit_code != 0 THEN ? -- failed + + WHEN hsi.install_script_exit_code IS NOT NULL AND + hsi.install_script_exit_code = 0 THEN ? -- installed + + WHEN hsi.install_script_exit_code IS NOT NULL AND + hsi.install_script_exit_code != 0 THEN ? -- failed + + WHEN hsi.pre_install_query_output IS NOT NULL AND + hsi.pre_install_query_output = '' THEN ? -- failed + + WHEN hsi.host_id IS NOT NULL THEN ? -- pending + + ELSE NULL -- not installed from Fleet installer + END AS status, + si.id AS installer_id -- NULL if no Fleet installer + FROM + software_titles st + LEFT OUTER JOIN + software_installers si ON st.id = si.title_id + LEFT OUTER JOIN + host_software_installs hsi ON si.id = hsi.software_installer_id AND hsi.host_id = ? + WHERE + -- use the latest install only + ( hsi.id IS NULL OR hsi.id = ( + SELECT hsi2.id + FROM host_software_installs hsi2 + WHERE hsi2.host_id = hsi.host_id AND hsi2.software_installer_id = hsi.software_installer_id + ORDER BY hsi2.created_at DESC + LIMIT 1 ) ) AND + -- software is installed on host + ( EXISTS ( + SELECT 1 + FROM + host_software hs + INNER JOIN + software s ON hs.software_id = s.id + WHERE + hs.host_id = ? AND + s.title_id = st.id + ) OR + -- or software install has been attempted on host + hsi.host_id IS NOT NULL ) +` + + const stmtAvailable = ` + SELECT + st.id, + st.name, + st.source, + si.filename as package_available_for_install, + NULL as last_install_installed_at, + NULL as last_install_install_uuid, + NULL as status, + si.id as installer_id + FROM + software_titles st + INNER JOIN + software_installers si ON st.id = si.title_id + WHERE + -- software is not installed on host, but is available in host's team + NOT EXISTS ( + SELECT 1 + FROM + host_software hs + INNER JOIN + software s ON hs.software_id = s.id + WHERE + hs.host_id = ? AND + s.title_id = st.id + ) AND + -- sofware install has not been attempted on host + NOT EXISTS ( + SELECT 1 + FROM + host_software_installs hsi + WHERE + hsi.host_id = ? AND + hsi.software_installer_id = si.id + ) AND + si.global_or_team_id = (SELECT COALESCE(h.team_id, 0) FROM hosts h WHERE h.id = ?) +` + + const selectColNames = ` + SELECT + id, + name, + source, + package_available_for_install, + last_install_installed_at, + last_install_install_uuid, + status, + CASE + WHEN status = ? THEN 4 -- failed + WHEN status = ? THEN 3 -- pending + WHEN status = ? THEN 2 -- installed + WHEN installer_id IS NOT NULL THEN 1 -- installer exists, not installed + ELSE 0 -- no installer exists + END AS status_sort +` + + args := []any{ + // for status_sort + fleet.SoftwareInstallerFailed, + fleet.SoftwareInstallerPending, + fleet.SoftwareInstallerInstalled, + + // for status + fleet.SoftwareInstallerInstalled, + fleet.SoftwareInstallerFailed, + fleet.SoftwareInstallerInstalled, + fleet.SoftwareInstallerFailed, + fleet.SoftwareInstallerFailed, + fleet.SoftwareInstallerPending, + + hostID, + hostID, + } + stmt := stmtInstalled + if includeAvailableForInstall { + stmt += ` UNION ` + stmtAvailable + args = append(args, hostID, hostID, hostID) + } + stmt = selectColNames + ` FROM ( ` + stmt + ` ) AS tbl ` + if opts.MatchQuery != "" { + stmt += " WHERE TRUE " // searchLike adds a "AND " + stmt, args = searchLike(stmt, args, opts.MatchQuery, "name") + } + + // apply default sort (adding source just to make it deterministic) + if opts.OrderKey == "" { + stmt += ` ORDER BY status_sort DESC, name ASC, source ASC ` + } + stmt, _ = appendListOptionsToSQL(stmt, &opts) + + type hostSoftware struct { + fleet.HostSoftwareWithInstaller + LastInstallInstalledAt *time.Time `db:"last_install_installed_at"` + LastInstallInstallUUID *string `db:"last_install_install_uuid"` + StatusSort sql.NullInt32 `db:"status_sort"` + } + var hostSoftwareList []*hostSoftware + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &hostSoftwareList, stmt, args...); err != nil { + return nil, nil, ctxerr.Wrap(ctx, err, "list host software") + } + + // collect the title ids to get the versions, vulnerabilities and installed + // paths for each software in the list. + titleIDs := make([]uint, 0, len(hostSoftwareList)) + byTitleID := make(map[uint]*hostSoftware, len(hostSoftwareList)) + for _, hs := range hostSoftwareList { + // promote the last install info to the proper destination fields + if hs.LastInstallInstallUUID != nil && *hs.LastInstallInstallUUID != "" { + hs.LastInstall = &fleet.HostSoftwareInstall{ + InstallUUID: *hs.LastInstallInstallUUID, + } + if hs.LastInstallInstalledAt != nil { + hs.LastInstall.InstalledAt = *hs.LastInstallInstalledAt + } + } + titleIDs = append(titleIDs, hs.ID) + byTitleID[hs.ID] = hs + } + + if len(titleIDs) > 0 { + // get the software versions installed on that host + const versionStmt = ` + SELECT + st.id as software_title_id, + s.id as software_id, + s.version, + hs.last_opened_at + FROM + software s + INNER JOIN + software_titles st ON s.title_id = st.id + INNER JOIN + host_software hs ON s.id = hs.software_id AND hs.host_id = ? + WHERE + st.id IN (?) +` + var installedVersions []*fleet.HostSoftwareInstalledVersion + stmt, args, err := sqlx.In(versionStmt, hostID, titleIDs) + if err != nil { + return nil, nil, ctxerr.Wrap(ctx, err, "building query args to list versions") + } + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &installedVersions, stmt, args...); err != nil { + return nil, nil, ctxerr.Wrap(ctx, err, "list software versions") + } + + // store the installed versions with the proper software entry and collect + // the software ids. + softwareIDs := make([]uint, 0, len(installedVersions)) + bySoftwareID := make(map[uint]*fleet.HostSoftwareInstalledVersion, len(hostSoftwareList)) + for _, ver := range installedVersions { + hs := byTitleID[ver.SoftwareTitleID] + hs.InstalledVersions = append(hs.InstalledVersions, ver) + softwareIDs = append(softwareIDs, ver.SoftwareID) + bySoftwareID[ver.SoftwareID] = ver + } + + if len(softwareIDs) > 0 { + const cveStmt = ` + SELECT + sc.software_id, + sc.cve + FROM + software_cve sc + WHERE + sc.software_id IN (?) + ORDER BY + software_id, cve + ` + type softwareCVE struct { + SoftwareID uint `db:"software_id"` + CVE string `db:"cve"` + } + var softwareCVEs []softwareCVE + stmt, args, err = sqlx.In(cveStmt, softwareIDs) + if err != nil { + return nil, nil, ctxerr.Wrap(ctx, err, "building query args to list cves") + } + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &softwareCVEs, stmt, args...); err != nil { + return nil, nil, ctxerr.Wrap(ctx, err, "list software cves") + } + + // store the CVEs with the proper software entry + for _, cve := range softwareCVEs { + ver := bySoftwareID[cve.SoftwareID] + ver.Vulnerabilities = append(ver.Vulnerabilities, cve.CVE) + } + + const pathsStmt = ` + SELECT + hsip.software_id, + hsip.installed_path + FROM + host_software_installed_paths hsip + WHERE + hsip.host_id = ? AND + hsip.software_id IN (?) + ORDER BY + software_id, installed_path + ` + type installedPath struct { + SoftwareID uint `db:"software_id"` + InstalledPath string `db:"installed_path"` + } + var installedPaths []installedPath + stmt, args, err = sqlx.In(pathsStmt, hostID, softwareIDs) + if err != nil { + return nil, nil, ctxerr.Wrap(ctx, err, "building query args to list installed paths") + } + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &installedPaths, stmt, args...); err != nil { + return nil, nil, ctxerr.Wrap(ctx, err, "list software installed paths") + } + + // store the installed paths with the proper software entry + for _, path := range installedPaths { + ver := bySoftwareID[path.SoftwareID] + ver.InstalledPaths = append(ver.InstalledPaths, path.InstalledPath) + } + } + } + + perPage := opts.PerPage + var metaData *fleet.PaginationMetadata + if opts.IncludeMetadata { + if perPage <= 0 { + perPage = defaultSelectLimit + } + metaData = &fleet.PaginationMetadata{HasPreviousResults: opts.Page > 0} + if len(hostSoftwareList) > int(perPage) { + metaData.HasNextResults = true + hostSoftwareList = hostSoftwareList[:len(hostSoftwareList)-1] + } + } + + software := make([]*fleet.HostSoftwareWithInstaller, 0, len(hostSoftwareList)) + for _, hs := range hostSoftwareList { + hs := hs + software = append(software, &hs.HostSoftwareWithInstaller) + } + return software, metaData, nil +} diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index ef6007510c..6b0709524b 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -16,6 +16,7 @@ import ( "github.com/fleetdm/fleet/v4/server/ptr" "github.com/fleetdm/fleet/v4/server/test" "github.com/fleetdm/fleet/v4/server/vulnerabilities/oval" + "github.com/google/uuid" "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -61,6 +62,7 @@ func TestSoftware(t *testing.T) { {"deleteHostSoftwareInstalledPaths", testDeleteHostSoftwareInstalledPaths}, {"insertHostSoftwareInstalledPaths", testInsertHostSoftwareInstalledPaths}, {"VerifySoftwareChecksum", testVerifySoftwareChecksum}, + {"ListHostSoftware", testListHostSoftware}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -2929,3 +2931,437 @@ func testVerifySoftwareChecksum(t *testing.T, ds *Datastore) { require.Equal(t, software[i], got) } } + +func testListHostSoftware(t *testing.T, ds *Datastore) { + ctx := context.Background() + host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) + otherHost := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now()) + opts := fleet.ListOptions{PerPage: 10, IncludeMetadata: true} + + // no software yet + sw, meta, err := ds.ListHostSoftware(ctx, host.ID, false, opts) + require.NoError(t, err) + require.Empty(t, sw) + require.Equal(t, &fleet.PaginationMetadata{}, meta) + + // works with available software too + sw, meta, err = ds.ListHostSoftware(ctx, host.ID, true, opts) + require.NoError(t, err) + require.Empty(t, sw) + require.Equal(t, &fleet.PaginationMetadata{}, meta) + + // add software to the host + software := []fleet.Software{ + {Name: "a", Version: "0.0.1", Source: "chrome_extensions"}, + {Name: "a", Version: "0.0.2", Source: "deb_packages"}, // different source, so different title than a-chrome + {Name: "b", Version: "0.0.3", Source: "apps"}, + {Name: "c", Version: "0.0.4", Source: "deb_packages"}, + {Name: "c", Version: "0.0.5", Source: "deb_packages"}, + {Name: "d", Version: "0.0.6", Source: "deb_packages"}, + } + mutationResults, err := ds.UpdateHostSoftware(ctx, host.ID, software) + require.NoError(t, err) + require.NoError(t, ds.LoadHostSoftware(ctx, host, false)) + + // add other software to the other host, won't be returned + otherSoftware := []fleet.Software{ + {Name: "a", Version: "0.0.7", Source: "chrome_extensions"}, + {Name: "f", Version: "0.0.8", Source: "chrome_extensions"}, + } + _, err = ds.UpdateHostSoftware(ctx, otherHost.ID, otherSoftware) + require.NoError(t, err) + + // add some vulnerabilities and installed paths + vulns := []fleet.SoftwareVulnerability{ + {SoftwareID: host.Software[0].ID, CVE: "CVE-a-0001"}, + {SoftwareID: host.Software[0].ID, CVE: "CVE-a-0002"}, + {SoftwareID: host.Software[0].ID, CVE: "CVE-a-0003"}, + {SoftwareID: host.Software[2].ID, CVE: "CVE-b-0001"}, + } + for _, v := range vulns { + _, err = ds.InsertSoftwareVulnerability(ctx, v, fleet.NVDSource) + require.NoError(t, err) + } + + swPaths := map[string]struct{}{} + installPaths := make([]string, 0, len(software)) + for _, s := range software { + path := fmt.Sprintf("/some/path/%s", s.Name) + key := fmt.Sprintf("%s%s%s", path, fleet.SoftwareFieldSeparator, s.ToUniqueStr()) + swPaths[key] = struct{}{} + installPaths = append(installPaths, path) + } + err = ds.UpdateHostSoftwareInstalledPaths(ctx, host.ID, swPaths, mutationResults) + require.NoError(t, err) + + err = ds.ReconcileSoftwareTitles(ctx) + require.NoError(t, err) + + expected := map[string]*fleet.HostSoftwareWithInstaller{ + "a1": {Name: software[0].Name, Source: software[0].Source, InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ + {Version: software[0].Version, Vulnerabilities: []string{vulns[0].CVE, vulns[1].CVE, vulns[2].CVE}, InstalledPaths: []string{installPaths[0]}}, + }}, + "a2": {Name: software[1].Name, Source: software[1].Source, InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ + {Version: software[1].Version, InstalledPaths: []string{installPaths[1]}}, + }}, + "b": {Name: software[2].Name, Source: software[2].Source, InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ + {Version: software[2].Version, Vulnerabilities: []string{vulns[3].CVE}, InstalledPaths: []string{installPaths[2]}}, + }}, + "c": {Name: software[3].Name, Source: software[3].Source, InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ + {Version: software[3].Version, InstalledPaths: []string{installPaths[3]}}, + {Version: software[4].Version, InstalledPaths: []string{installPaths[4]}}, + }}, + "d": {Name: software[5].Name, Source: software[5].Source, InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ + {Version: software[5].Version, InstalledPaths: []string{installPaths[5]}}, + }}, + } + + compareResults := func(expected, got []*fleet.HostSoftwareWithInstaller) { + require.Len(t, got, len(expected)) + // clear ids and timestamps for comparison + for _, g := range got { + g.ID = 0 + if g.LastInstall != nil { + g.LastInstall.InstalledAt = time.Time{} + } + for _, v := range g.InstalledVersions { + v.SoftwareID = 0 + v.SoftwareTitleID = 0 + } + } + require.Equal(t, expected, got) + } + + // it now returns the software with vulnerabilities and installed paths + sw, meta, err = ds.ListHostSoftware(ctx, host.ID, false, opts) + require.NoError(t, err) + require.Equal(t, &fleet.PaginationMetadata{}, meta) + compareResults([]*fleet.HostSoftwareWithInstaller{ + expected["a1"], expected["a2"], expected["b"], expected["c"], expected["d"], + }, sw) + + // create some Fleet installers and map them to a software title, + // including one for a team + tm, err := ds.NewTeam(ctx, &fleet.Team{Name: "team1"}) + require.NoError(t, err) + + var swi1Pending, swi2Installed, swi3Failed, swi4Available, swi5Tm uint + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + // keep title id of software B, will use it to associate an installer with it + var swbTitleID uint + err := sqlx.GetContext(ctx, q, &swbTitleID, `SELECT id FROM software_titles WHERE name = 'b' AND source = 'apps'`) + if err != nil { + return err + } + + // create the install script content (same for all installers, doesn't matter) + installScript := `echo 'foo'` + res, err := q.ExecContext(ctx, `INSERT INTO script_contents (md5_checksum, contents) VALUES (UNHEX(md5(?)), ?)`, installScript, installScript) + if err != nil { + return err + } + scriptContentID, _ := res.LastInsertId() + + // create software titles for all but swi1Pending (will be linked to + // existing software title b) + var titleIDs []uint + for i := 0; i < 4; i++ { + res, err := q.ExecContext(ctx, `INSERT INTO software_titles (name, source) VALUES (?, 'apps')`, fmt.Sprintf("i%d", i)) + if err != nil { + return err + } + id, _ := res.LastInsertId() + titleIDs = append(titleIDs, uint(id)) + } + + var swiIDs []uint + for i := 0; i < 5; i++ { + var ( + titleID uint + teamID *uint + globalOrTeamID uint + ) + if i == 0 { + titleID = swbTitleID + } else { + titleID = titleIDs[i-1] + } + if i == 4 { + teamID = &tm.ID + globalOrTeamID = tm.ID + } + res, err := q.ExecContext(ctx, ` + INSERT INTO software_installers + (team_id, global_or_team_id, title_id, filename, version, install_script_content_id, storage_id) + VALUES + (?, ?, ?, ?, ?, ?, unhex(?))`, + teamID, globalOrTeamID, titleID, fmt.Sprintf("installer-%d.pkg", i), fmt.Sprintf("v%d.0.0", i), scriptContentID, hex.EncodeToString([]byte("test"))) + if err != nil { + return err + } + id, _ := res.LastInsertId() + swiIDs = append(swiIDs, uint(id)) + } + swi1Pending, swi2Installed, swi3Failed, swi4Available, swi5Tm = swiIDs[0], swiIDs[1], swiIDs[2], swiIDs[3], swiIDs[4] + + // create the results for the host + + // swi1 is pending (all results are NULL) + _, err = q.ExecContext(ctx, ` + INSERT INTO host_software_installs (execution_id, host_id, software_installer_id) VALUES (?, ?, ?)`, + "uuid1", host.ID, swi1Pending) + if err != nil { + return err + } + + // swi2 is installed + _, err = q.ExecContext(ctx, ` + INSERT INTO host_software_installs (execution_id, host_id, software_installer_id, pre_install_query_output, install_script_exit_code, post_install_script_exit_code) + VALUES (?, ?, ?, ?, ?, ?)`, + "uuid2", host.ID, swi2Installed, "ok", 0, 0) + if err != nil { + return err + } + + // swi3 is failed, also add an install request on the other host + _, err = q.ExecContext(ctx, ` + INSERT INTO host_software_installs (execution_id, host_id, software_installer_id, pre_install_query_output, install_script_exit_code) + VALUES (?, ?, ?, ?, ?)`, + "uuid3", host.ID, swi3Failed, "ok", 1) + if err != nil { + return err + } + _, err = q.ExecContext(ctx, ` + INSERT INTO host_software_installs (execution_id, host_id, software_installer_id) VALUES (?, ?, ?)`, + uuid.NewString(), otherHost.ID, swi3Failed) + if err != nil { + return err + } + + // swi4 is available (no install request), but add a pending request on the other host + _, err = q.ExecContext(ctx, ` + INSERT INTO host_software_installs (execution_id, host_id, software_installer_id) VALUES (?, ?, ?)`, + uuid.NewString(), otherHost.ID, swi4Available) + if err != nil { + return err + } + + // swi5 is for another team + _ = swi5Tm + + return nil + }) + + // swi1Pending uses software title id of "b" + expected["b"] = &fleet.HostSoftwareWithInstaller{ + Name: "b", + Source: "apps", + Status: &fleet.SoftwareInstallerPending, + LastInstall: &fleet.HostSoftwareInstall{InstallUUID: "uuid1"}, + PackageAvailableForInstall: ptr.String("installer-0.pkg"), + InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ + {Version: software[2].Version, Vulnerabilities: []string{vulns[3].CVE}, InstalledPaths: []string{installPaths[2]}}, + }, + } + expected["i0"] = &fleet.HostSoftwareWithInstaller{ + Name: "i0", + Source: "apps", + Status: &fleet.SoftwareInstallerInstalled, + LastInstall: &fleet.HostSoftwareInstall{InstallUUID: "uuid2"}, + PackageAvailableForInstall: ptr.String("installer-1.pkg"), + } + expected["i1"] = &fleet.HostSoftwareWithInstaller{ + Name: "i1", + Source: "apps", + Status: &fleet.SoftwareInstallerFailed, + LastInstall: &fleet.HostSoftwareInstall{InstallUUID: "uuid3"}, + PackageAvailableForInstall: ptr.String("installer-2.pkg"), + } + expected["i2"] = &fleet.HostSoftwareWithInstaller{ + Name: "i2", + Source: "apps", + Status: nil, + LastInstall: nil, + PackageAvailableForInstall: ptr.String("installer-3.pkg"), + } + expected["i3"] = &fleet.HostSoftwareWithInstaller{ + Name: "i3", + Source: "apps", + Status: nil, + LastInstall: nil, + PackageAvailableForInstall: ptr.String("installer-4.pkg"), + } + + // request without available software, returns failed first, pending, installed, other + sw, meta, err = ds.ListHostSoftware(ctx, host.ID, false, opts) + require.NoError(t, err) + require.Equal(t, &fleet.PaginationMetadata{}, meta) + compareResults([]*fleet.HostSoftwareWithInstaller{ + expected["i1"], expected["b"], expected["i0"], expected["a1"], expected["a2"], expected["c"], expected["d"], + }, sw) + + // request with available software, returns failed first, pending, installed, available, other + sw, meta, err = ds.ListHostSoftware(ctx, host.ID, true, opts) + require.NoError(t, err) + require.Equal(t, &fleet.PaginationMetadata{}, meta) + compareResults([]*fleet.HostSoftwareWithInstaller{ + expected["i1"], expected["b"], expected["i0"], expected["i2"], expected["a1"], expected["a2"], expected["c"], expected["d"], + }, sw) + + // record a new install request for i1, this time as pending, and mark install request for b (swi1) as failed + time.Sleep(time.Second) // ensure the timestamp is later + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + // swi1 is now failed + _, err = q.ExecContext(ctx, ` + UPDATE host_software_installs SET install_script_exit_code = 2 WHERE execution_id = 'uuid1'`) + if err != nil { + return err + } + + // swi3 has a new install request pending + _, err = q.ExecContext(ctx, ` + INSERT INTO host_software_installs (execution_id, host_id, software_installer_id) + VALUES (?, ?, ?)`, + "uuid4", host.ID, swi3Failed) + if err != nil { + return err + } + return nil + }) + + expected["b"] = &fleet.HostSoftwareWithInstaller{ + Name: "b", + Source: "apps", + Status: &fleet.SoftwareInstallerFailed, + LastInstall: &fleet.HostSoftwareInstall{InstallUUID: "uuid1"}, + PackageAvailableForInstall: ptr.String("installer-0.pkg"), + InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ + {Version: software[2].Version, Vulnerabilities: []string{vulns[3].CVE}, InstalledPaths: []string{installPaths[2]}}, + }, + } + expected["i1"] = &fleet.HostSoftwareWithInstaller{ + Name: "i1", + Source: "apps", + Status: &fleet.SoftwareInstallerPending, + LastInstall: &fleet.HostSoftwareInstall{InstallUUID: "uuid4"}, + PackageAvailableForInstall: ptr.String("installer-2.pkg"), + } + + // request without available software, returns failed first, pending, installed, other + sw, meta, err = ds.ListHostSoftware(ctx, host.ID, false, opts) + require.NoError(t, err) + require.Equal(t, &fleet.PaginationMetadata{}, meta) + compareResults([]*fleet.HostSoftwareWithInstaller{ + expected["b"], expected["i1"], expected["i0"], expected["a1"], expected["a2"], expected["c"], expected["d"], + }, sw) + + // request with available software, returns failed first, pending, installed, available, other + sw, meta, err = ds.ListHostSoftware(ctx, host.ID, true, opts) + require.NoError(t, err) + require.Equal(t, &fleet.PaginationMetadata{}, meta) + compareResults([]*fleet.HostSoftwareWithInstaller{ + expected["b"], expected["i1"], expected["i0"], expected["i2"], expected["a1"], expected["a2"], expected["c"], expected["d"], + }, sw) + + // create a new host in the team, with no software + tmHost := test.NewHost(t, ds, "host3", "", "host3key", "host3uuid", time.Now()) + err = ds.AddHostsToTeam(ctx, &tm.ID, []uint{tmHost.ID}) + require.NoError(t, err) + + // no installed software for this host + sw, meta, err = ds.ListHostSoftware(ctx, tmHost.ID, false, opts) + require.NoError(t, err) + require.Empty(t, sw) + require.Equal(t, &fleet.PaginationMetadata{}, meta) + + // sees the available installer in its team + sw, _, err = ds.ListHostSoftware(ctx, tmHost.ID, true, opts) + require.NoError(t, err) + compareResults([]*fleet.HostSoftwareWithInstaller{expected["i3"]}, sw) + + // test with a search query (searches on name), with and without available software + opts.MatchQuery = "a" + sw, _, err = ds.ListHostSoftware(ctx, host.ID, false, opts) + require.NoError(t, err) + compareResults([]*fleet.HostSoftwareWithInstaller{expected["a1"], expected["a2"]}, sw) + sw, _, err = ds.ListHostSoftware(ctx, host.ID, true, opts) + require.NoError(t, err) + compareResults([]*fleet.HostSoftwareWithInstaller{expected["a1"], expected["a2"]}, sw) + + opts.MatchQuery = "zz" + sw, _, err = ds.ListHostSoftware(ctx, host.ID, false, opts) + require.NoError(t, err) + require.Empty(t, sw) + sw, _, err = ds.ListHostSoftware(ctx, host.ID, true, opts) + require.NoError(t, err) + require.Empty(t, sw) + + // test the pagination + cases := []struct { + opts fleet.ListOptions + withAvailable bool + wantNames []string + wantMeta *fleet.PaginationMetadata + }{ + { + opts: fleet.ListOptions{PerPage: 3}, + withAvailable: false, + wantNames: []string{expected["b"].Name, expected["i1"].Name, expected["i0"].Name}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false}, + }, + { + opts: fleet.ListOptions{Page: 1, PerPage: 3}, + withAvailable: false, + wantNames: []string{expected["a1"].Name, expected["a2"].Name, expected["c"].Name}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: true}, + }, + { + opts: fleet.ListOptions{Page: 2, PerPage: 3}, + withAvailable: false, + wantNames: []string{expected["d"].Name}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true}, + }, + { + opts: fleet.ListOptions{Page: 3, PerPage: 3}, + withAvailable: false, + wantNames: []string{}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true}, + }, + { + opts: fleet.ListOptions{PerPage: 4}, + withAvailable: true, + wantNames: []string{expected["b"].Name, expected["i1"].Name, expected["i0"].Name, expected["i2"].Name}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false}, + }, + { + opts: fleet.ListOptions{Page: 1, PerPage: 4}, + withAvailable: true, + wantNames: []string{expected["a1"].Name, expected["a2"].Name, expected["c"].Name, expected["d"].Name}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true}, + }, + { + opts: fleet.ListOptions{Page: 2, PerPage: 4}, + withAvailable: true, + wantNames: []string{}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true}, + }, + } + for _, c := range cases { + t.Run(fmt.Sprintf("%t: %#v", c.withAvailable, c.opts), func(t *testing.T) { + // always include metadata + c.opts.IncludeMetadata = true + + sw, meta, err := ds.ListHostSoftware(ctx, host.ID, c.withAvailable, c.opts) + require.NoError(t, err) + + require.Equal(t, len(c.wantNames), len(sw)) + require.Equal(t, c.wantMeta, meta) + + names := make([]string, 0, len(sw)) + for _, s := range sw { + names = append(names, s.Name) + } + require.Equal(t, c.wantNames, names) + }) + } +} diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index d5d1a26d6a..c171fc1cdc 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -544,6 +544,8 @@ type Datastore interface { InsertCVEMeta(ctx context.Context, cveMeta []CVEMeta) error ListCVEs(ctx context.Context, maxAge time.Duration) ([]CVEMeta, error) + ListHostSoftware(ctx context.Context, hostID uint, includeAvailableForInstall bool, opts ListOptions) ([]*HostSoftwareWithInstaller, *PaginationMetadata, error) + /////////////////////////////////////////////////////////////////////////////// // OperatingSystemsStore diff --git a/server/fleet/service.go b/server/fleet/service.go index d6aa85f0ec..8cf2779191 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -411,6 +411,10 @@ type Service interface { // OSVersion returns an operating system and associated host counts OSVersion(ctx context.Context, osVersionID uint, teamID *uint, includeCVSS bool) (*OSVersion, *time.Time, error) + // ListHostSoftware lists the software installed or available for install on + // the specified host. + ListHostSoftware(ctx context.Context, hostID uint, opts ListOptions) ([]*HostSoftwareWithInstaller, *PaginationMetadata, error) + // ///////////////////////////////////////////////////////////////////////////// // AppConfigService provides methods for configuring the Fleet application diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index e911b6a70c..bdc921dd20 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -4,6 +4,7 @@ import ( "context" "errors" "io" + "time" ) // SoftwareInstallerStore is the interface to store and retrieve software @@ -103,3 +104,38 @@ type HostSoftwareInstallerResult struct { // PostInstallScriptOutput is the output of the post-install script on the host. PostInstallScriptOutput string `json:"post_install_script_output" db:"post_install_script_output"` } + +// HostSoftwareWithInstaller represents the list of software installed on a +// host with installer information if a matching installer exists. This is the +// payload returned by the "Get host's (device's) software" endpoints. +type HostSoftwareWithInstaller struct { + ID uint `json:"id" db:"id"` + Name string `json:"name" db:"name"` + Source string `json:"source" db:"source"` + Status *SoftwareInstallerStatus `json:"status" db:"status"` + LastInstall *HostSoftwareInstall `json:"last_install"` + InstalledVersions []*HostSoftwareInstalledVersion `json:"installed_versions"` + + // PackageAvailableForInstall is only present for the user-authenticated + // endpoint, not the device-authenticated one. I.e. when + // available-but-not-installed software are part of the response. + PackageAvailableForInstall *string `json:"package_available_for_install,omitempty" db:"package_available_for_install"` +} + +// HostSoftwareInstall represents installation of software on a host from a +// Fleet software installer. +type HostSoftwareInstall struct { + InstallUUID string `json:"install_uuid" db:"install_id"` + InstalledAt time.Time `json:"installed_at" db:"installed_at"` +} + +// HostSoftwareInstalledVersion represents a version of software installed on a +// host. +type HostSoftwareInstalledVersion struct { + SoftwareID uint `json:"-" db:"software_id"` + SoftwareTitleID uint `json:"-" db:"software_title_id"` + Version string `json:"version" db:"version"` + LastOpenedAt *time.Time `json:"last_opened_at" db:"last_opened_at"` + Vulnerabilities []string `json:"vulnerabilities" db:"vulnerabilities"` + InstalledPaths []string `json:"installed_paths" db:"installed_paths"` +} diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 7f14f0d000..d415ffecfa 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -399,6 +399,8 @@ type InsertCVEMetaFunc func(ctx context.Context, cveMeta []fleet.CVEMeta) error type ListCVEsFunc func(ctx context.Context, maxAge time.Duration) ([]fleet.CVEMeta, error) +type ListHostSoftwareFunc func(ctx context.Context, hostID uint, includeAvailableForInstall bool, opts fleet.ListOptions) ([]*fleet.HostSoftwareWithInstaller, *fleet.PaginationMetadata, error) + type GetHostOperatingSystemFunc func(ctx context.Context, hostID uint) (*fleet.OperatingSystem, error) type ListOperatingSystemsFunc func(ctx context.Context) ([]fleet.OperatingSystem, error) @@ -1490,6 +1492,9 @@ type DataStore struct { ListCVEsFunc ListCVEsFunc ListCVEsFuncInvoked bool + ListHostSoftwareFunc ListHostSoftwareFunc + ListHostSoftwareFuncInvoked bool + GetHostOperatingSystemFunc GetHostOperatingSystemFunc GetHostOperatingSystemFuncInvoked bool @@ -3603,6 +3608,13 @@ func (s *DataStore) ListCVEs(ctx context.Context, maxAge time.Duration) ([]fleet return s.ListCVEsFunc(ctx, maxAge) } +func (s *DataStore) ListHostSoftware(ctx context.Context, hostID uint, includeAvailableForInstall bool, opts fleet.ListOptions) ([]*fleet.HostSoftwareWithInstaller, *fleet.PaginationMetadata, error) { + s.mu.Lock() + s.ListHostSoftwareFuncInvoked = true + s.mu.Unlock() + return s.ListHostSoftwareFunc(ctx, hostID, includeAvailableForInstall, opts) +} + func (s *DataStore) GetHostOperatingSystem(ctx context.Context, hostID uint) (*fleet.OperatingSystem, error) { s.mu.Lock() s.GetHostOperatingSystemFuncInvoked = true diff --git a/server/service/devices.go b/server/service/devices.go index 7f8efe4eb5..d0a319b8b1 100644 --- a/server/service/devices.go +++ b/server/service/devices.go @@ -590,3 +590,42 @@ func migrateMDMDeviceEndpoint(ctx context.Context, request interface{}, svc flee func (svc *Service) TriggerMigrateMDMDevice(ctx context.Context, host *fleet.Host) error { return fleet.ErrMissingLicense } + +//////////////////////////////////////////////////////////////////////////////// +// Get Current Device's Software +//////////////////////////////////////////////////////////////////////////////// + +type getDeviceSoftwareRequest struct { + Token string `url:"token"` + ListOptions fleet.ListOptions `url:"list_options"` +} + +func (r *getDeviceSoftwareRequest) deviceAuthToken() string { + return r.Token +} + +type getDeviceSoftwareResponse struct { + Software []*fleet.HostSoftwareWithInstaller `json:"software"` + Meta *fleet.PaginationMetadata `json:"meta,omitempty"` + Err error `json:"error,omitempty"` +} + +func (r getDeviceSoftwareResponse) error() error { return r.Err } + +func getDeviceSoftwareEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { + host, ok := hostctx.FromContext(ctx) + if !ok { + err := ctxerr.Wrap(ctx, fleet.NewAuthRequiredError("internal error: missing host from request context")) + return getDeviceSoftwareResponse{Err: err}, nil + } + + req := request.(*getDeviceSoftwareRequest) + res, meta, err := svc.ListHostSoftware(ctx, host.ID, req.ListOptions) + if err != nil { + return getDeviceSoftwareResponse{Err: err}, nil + } + if res == nil { + res = []*fleet.HostSoftwareWithInstaller{} + } + return getDeviceSoftwareResponse{Software: res, Meta: meta}, nil +} diff --git a/server/service/handler.go b/server/service/handler.go index dddc850aa3..3a5fb87051 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -398,6 +398,7 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC ue.GET("/api/_version_/fleet/hosts/{id:[0-9]+}/health", getHostHealthEndpoint, getHostHealthRequest{}) ue.POST("/api/_version_/fleet/hosts/{id:[0-9]+}/labels", addLabelsToHostEndpoint, addLabelsToHostRequest{}) ue.DELETE("/api/_version_/fleet/hosts/{id:[0-9]+}/labels", removeLabelsFromHostEndpoint, removeLabelsFromHostRequest{}) + ue.GET("/api/_version_/fleet/hosts/{id:[0-9]+}/software", getHostSoftwareEndpoint, getHostSoftwareRequest{}) ue.GET("/api/_version_/fleet/hosts/summary/mdm", getHostMDMSummary, getHostMDMSummaryRequest{}) ue.GET("/api/_version_/fleet/hosts/{id:[0-9]+}/mdm", getHostMDM, getHostMDMRequest{}) @@ -753,6 +754,9 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC de.WithCustomMiddleware( errorLimiter.Limit("send_device_error", desktopQuota), ).POST("/api/_version_/fleet/device/{token}/debug/errors", fleetdError, fleetdErrorRequest{}) + de.WithCustomMiddleware( + errorLimiter.Limit("get_device_software", desktopQuota), + ).GET("/api/_version_/fleet/device/{token}/software", getDeviceSoftwareEndpoint, getDeviceSoftwareRequest{}) // mdm-related endpoints available via device authentication demdm := de.WithCustomMiddleware(mdmConfiguredMiddleware.VerifyAppleMDM()) diff --git a/server/service/hosts.go b/server/service/hosts.go index e08bf96ad4..23d2e96247 100644 --- a/server/service/hosts.go +++ b/server/service/hosts.go @@ -2467,3 +2467,72 @@ func (svc *Service) validateLabelNames(ctx context.Context, action string, label return labelIDs, nil } + +//////////////////////////////////////////////////////////////////////////////// +// Host Software +//////////////////////////////////////////////////////////////////////////////// + +type getHostSoftwareRequest struct { + ID uint `url:"id"` + ListOptions fleet.ListOptions `url:"list_options"` +} + +type getHostSoftwareResponse struct { + Software []*fleet.HostSoftwareWithInstaller `json:"software"` + Meta *fleet.PaginationMetadata `json:"meta,omitempty"` + Err error `json:"error,omitempty"` +} + +func (r getHostSoftwareResponse) error() error { return r.Err } + +func getHostSoftwareEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { + req := request.(*getHostSoftwareRequest) + res, meta, err := svc.ListHostSoftware(ctx, req.ID, req.ListOptions) + if err != nil { + return getHostSoftwareResponse{Err: err}, nil + } + if res == nil { + res = []*fleet.HostSoftwareWithInstaller{} + } + return getHostSoftwareResponse{Software: res, Meta: meta}, nil +} + +func (svc *Service) ListHostSoftware(ctx context.Context, hostID uint, opts fleet.ListOptions) ([]*fleet.HostSoftwareWithInstaller, *fleet.PaginationMetadata, error) { + // if the request is token-authenticated ("My device" page), we don't include software + // that is not installed but for which there's an installer available for that host. + var includeAvailableForInstall bool + + if !svc.authz.IsAuthenticatedWith(ctx, authzctx.AuthnDeviceToken) { + includeAvailableForInstall = true + + if err := svc.authz.Authorize(ctx, &fleet.Host{}, fleet.ActionList); err != nil { + return nil, nil, err + } + + host, err := svc.ds.HostLite(ctx, hostID) + if err != nil { + return nil, nil, ctxerr.Wrap(ctx, err, "get host lite") + } + + // Authorize again with team loaded now that we have team_id + if err := svc.authz.Authorize(ctx, host, fleet.ActionRead); err != nil { + return nil, nil, err + } + } + + // cursor-based pagination is not supported + opts.After = "" + // custom ordering is not supported + opts.OrderKey = "" + // always include metadata + opts.IncludeMetadata = true + + software, meta, err := svc.ds.ListHostSoftware(ctx, hostID, includeAvailableForInstall, opts) + if !includeAvailableForInstall { + // for the device page, we don't want to return the package name + for _, s := range software { + s.PackageAvailableForInstall = nil + } + } + return software, meta, ctxerr.Wrap(ctx, err, "list host software") +} diff --git a/server/service/hosts_test.go b/server/service/hosts_test.go index b2480bf3ab..8c556d1692 100644 --- a/server/service/hosts_test.go +++ b/server/service/hosts_test.go @@ -618,6 +618,9 @@ func TestHostAuth(t *testing.T) { ds.GetHostLockWipeStatusFunc = func(ctx context.Context, host *fleet.Host) (*fleet.HostLockWipeStatus, error) { return &fleet.HostLockWipeStatus{}, nil } + ds.ListHostSoftwareFunc = func(ctx context.Context, hostID uint, includeAvailableForInstall bool, opts fleet.ListOptions) ([]*fleet.HostSoftwareWithInstaller, *fleet.PaginationMetadata, error) { + return nil, nil, nil + } testCases := []struct { name string @@ -759,6 +762,12 @@ func TestHostAuth(t *testing.T) { _, err = svc.SetCustomHostDeviceMapping(ctx, 2, "a@b.c") checkAuthErr(t, tt.shouldFailGlobalWrite, err) + + _, _, err = svc.ListHostSoftware(ctx, 1, fleet.ListOptions{}) + checkAuthErr(t, tt.shouldFailTeamRead, err) + + _, _, err = svc.ListHostSoftware(ctx, 2, fleet.ListOptions{}) + checkAuthErr(t, tt.shouldFailGlobalRead, err) }) } diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index b50bd25c7b..5b1b9b51f1 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -8596,6 +8596,55 @@ func (s *integrationEnterpriseTestSuite) TestCalendarEventsTransferringHosts() { require.True(t, fleet.IsNotFound(err)) } +func (s *integrationEnterpriseTestSuite) TestListHostSoftware() { + ctx := context.Background() + t := s.T() + + token := "good_token" + host := createHostAndDeviceToken(t, s.ds, token) + + // create some software for that host + software := []fleet.Software{ + {Name: "foo", Version: "0.0.1", Source: "chrome_extensions"}, + {Name: "foo", Version: "0.0.2", Source: "chrome_extensions"}, + {Name: "bar", Version: "0.0.1", Source: "apps"}, + } + _, err := s.ds.UpdateHostSoftware(ctx, host.ID, software) + require.NoError(t, err) + err = s.ds.ReconcileSoftwareTitles(ctx) + require.NoError(t, err) + + var getHostSw getHostSoftwareResponse + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", host.ID), nil, http.StatusOK, &getHostSw) + require.Len(t, getHostSw.Software, 2) // foo and bar + require.Equal(t, getHostSw.Software[0].Name, "bar") + require.Equal(t, getHostSw.Software[1].Name, "foo") + require.Len(t, getHostSw.Software[1].InstalledVersions, 2) + + var getDeviceSw getDeviceSoftwareResponse + res := s.DoRawNoAuth("GET", "/api/latest/fleet/device/"+token+"/software", nil, http.StatusOK) + err = json.NewDecoder(res.Body).Decode(&getDeviceSw) + require.NoError(t, err) + require.Len(t, getDeviceSw.Software, 2) // foo and bar + require.Equal(t, getDeviceSw.Software[0].Name, "bar") + require.Equal(t, getDeviceSw.Software[1].Name, "foo") + require.Len(t, getDeviceSw.Software[1].InstalledVersions, 2) + + // test with a query + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", host.ID), nil, http.StatusOK, &getHostSw, "query", "foo") + require.Len(t, getHostSw.Software, 1) // foo only + require.Equal(t, getHostSw.Software[0].Name, "foo") + require.Len(t, getHostSw.Software[0].InstalledVersions, 2) + res = s.DoRawNoAuth("GET", "/api/latest/fleet/device/"+token+"/software?query=bar", nil, http.StatusOK) + err = json.NewDecoder(res.Body).Decode(&getDeviceSw) + require.NoError(t, err) + require.Len(t, getDeviceSw.Software, 1) // bar only + require.Equal(t, getDeviceSw.Software[0].Name, "bar") + require.Len(t, getDeviceSw.Software[0].InstalledVersions, 1) + + // TODO(mna): more advanced integration tests with Software Installers once the APIs are in place. +} + func genDistributedReqWithPolicyResults(host *fleet.Host, policyResults map[uint]*bool) submitDistributedQueryResultsRequestShim { var ( results = make(map[string]json.RawMessage) From 8f5b3a872db5faf1b067804ae7af5deeb4122b6d Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Thu, 2 May 2024 08:20:54 -0500 Subject: [PATCH 11/56] Add backend to upload/delete software installers (#18660) Issue #18320 --- ee/server/service/software_installers.go | 110 ++++++++++++ server/datastore/mysql/scripts.go | 18 ++ server/datastore/mysql/software.go | 3 +- server/datastore/mysql/software_installers.go | 136 ++++++++++++++ server/fleet/datastore.go | 13 ++ server/fleet/service.go | 7 + server/fleet/software_installer.go | 48 ++++- server/mock/datastore_mock.go | 36 ++++ server/service/handler.go | 4 + server/service/integration_mdm_test.go | 168 ++++++++++++++++++ server/service/software_installers.go | 142 +++++++++++++++ .../testdata/software-installers/ruby.deb | Bin 0 -> 11340 bytes 12 files changed, 677 insertions(+), 8 deletions(-) create mode 100644 ee/server/service/software_installers.go create mode 100644 server/datastore/mysql/software_installers.go create mode 100644 server/service/software_installers.go create mode 100644 server/service/testdata/software-installers/ruby.deb diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go new file mode 100644 index 0000000000..d70bda7f40 --- /dev/null +++ b/ee/server/service/software_installers.go @@ -0,0 +1,110 @@ +package service + +import ( + "context" + "encoding/hex" + "path/filepath" + "strings" + + "github.com/fleetdm/fleet/v4/pkg/file" + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/go-kit/kit/log/level" +) + +func (svc *Service) UploadSoftwareInstaller(ctx context.Context, payload *fleet.UploadSoftwareInstallerPayload) error { + if err := svc.authz.Authorize(ctx, &fleet.SoftwareInstaller{TeamID: payload.TeamID}, fleet.ActionWrite); err != nil { + return err + } + if payload == nil { + return ctxerr.New(ctx, "payload is required") + } + + if payload.InstallerFile == nil { + return ctxerr.New(ctx, "installer file is required") + } + + title, vers, hash, err := file.ExtractInstallerMetadata(payload.Filename, payload.InstallerFile) + if err != nil { + // TODO: confirm error handling + if strings.Contains(err.Error(), "unsupported file type") { + return &fleet.BadRequestError{ + Message: "The file should be .pkg, .msi, .exe or .deb.", + InternalErr: ctxerr.Wrap(ctx, err, "extracting metadata from installer"), + } + } + return ctxerr.Wrap(ctx, err, "extracting metadata from installer") + } + payload.Title = title + payload.Version = vers + payload.StorageID = hex.EncodeToString(hash) + + // checck if exists in the installer store + exists, err := svc.softwareInstallStore.Exists(ctx, payload.StorageID) + if err != nil { + return ctxerr.Wrap(ctx, err, "checking if installer exists") + } + if !exists { + // reset the reader before storing (it was consumed to extract metadata) + if _, err := payload.InstallerFile.Seek(0, 0); err != nil { + return ctxerr.Wrap(ctx, err, "resetting installer file reader") + } + if err := svc.softwareInstallStore.Put(ctx, payload.StorageID, payload.InstallerFile); err != nil { + return ctxerr.Wrap(ctx, err, "storing installer") + } + } + + if payload.InstallScript == "" { + installerType := file.InstallerType(strings.TrimPrefix(filepath.Ext(payload.Filename), ".")) + installerPath := "some path" // TODO: where does this come from? + payload.InstallScript = file.GetInstallScript(installerType, installerPath) + } + + // TODO: basic validation of install and post-install script (e.g., supported interpreters)? + + // TODO: any validation of pre-install query? + + source, err := fleet.SofwareInstallerSourceFromFilename(payload.Filename) + if err != nil { + return ctxerr.Wrap(ctx, err, "determining source from filename") + } + payload.Source = source + + installerID, err := svc.ds.MatchOrCreateSoftwareInstaller(ctx, payload) + if err != nil { + return ctxerr.Wrap(ctx, err, "matching or creating software installer") + } + level.Debug(svc.logger).Log("msg", "software installer uploaded", "installer_id", installerID) + + // TODO: QA what breaks when you have a software title with no versions? + + return nil +} + +func (svc *Service) DeleteSoftwareInstaller(ctx context.Context, id uint) error { + // get the software installer to have its team id + meta, err := svc.ds.GetSoftwareInstallerMetadata(ctx, id) + if err != nil { + if fleet.IsNotFound(err) { + // couldn't get the metadata to have its team, authorize with a no-team + // as a fallback - the requested installer does not exist so there's + // no way to know what team it would be for, and returning a 404 without + // authorization would leak the existing/non existing ids. + if err := svc.authz.Authorize(ctx, &fleet.SoftwareInstaller{}, fleet.ActionWrite); err != nil { + return err + } + return ctxerr.Wrap(ctx, err, "getting software installer metadata") + } + } + + // do the actual authorization with the software installer's team id + if err := svc.authz.Authorize(ctx, &fleet.SoftwareInstaller{TeamID: meta.TeamID}, fleet.ActionWrite); err != nil { + return err + } + + if err := svc.ds.DeleteSoftwareInstaller(ctx, id); err != nil { + return ctxerr.Wrap(ctx, err, "deleting software installer") + } + + return nil +} diff --git a/server/datastore/mysql/scripts.go b/server/datastore/mysql/scripts.go index 24d2730fb9..97305ff1c8 100644 --- a/server/datastore/mysql/scripts.go +++ b/server/datastore/mysql/scripts.go @@ -1093,3 +1093,21 @@ WHERE } return nil } + +func (ds *Datastore) getOrGenerateScriptContentsID(ctx context.Context, contents string) (uint, error) { + csum := md5ChecksumScriptContent(contents) + scriptContentsID, err := ds.optimisticGetOrInsert(ctx, + ¶meterizedStmt{ + Statement: `SELECT id FROM script_contents WHERE md5_checksum = UNHEX(?)`, + Args: []interface{}{csum}, + }, + ¶meterizedStmt{ + Statement: `INSERT INTO script_contents (md5_checksum, contents) VALUES (UNHEX(?), ?)`, + Args: []interface{}{csum, contents}, + }, + ) + if err != nil { + return 0, err + } + return scriptContentsID, nil +} diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 532e473c6d..72f2b9b2c4 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -1058,7 +1058,6 @@ func (ds *Datastore) ListSoftwareCPEs(ctx context.Context) ([]fleet.SoftwareCPE, stmt := `SELECT id, software_id, cpe FROM software_cpe` err = sqlx.SelectContext(ctx, ds.reader(ctx), &result, stmt, args...) - if err != nil { return nil, ctxerr.Wrap(ctx, err, "loads cpes") } @@ -1423,7 +1422,7 @@ WHERE cleanupStmt := ` DELETE st FROM software_titles st LEFT JOIN software s ON s.title_id = st.id - WHERE s.title_id IS NULL` + WHERE s.title_id IS NULL AND NOT EXISTS (SELECT 1 FROM software_installers si WHERE si.title_id = st.id)` res, err = ds.writer(ctx).ExecContext(ctx, cleanupStmt) if err != nil { diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go new file mode 100644 index 0000000000..1027c6fe11 --- /dev/null +++ b/server/datastore/mysql/software_installers.go @@ -0,0 +1,136 @@ +package mysql + +import ( + "context" + "database/sql" + + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/jmoiron/sqlx" +) + +func (ds *Datastore) MatchOrCreateSoftwareInstaller(ctx context.Context, payload *fleet.UploadSoftwareInstallerPayload) (uint, error) { + titleID, err := ds.getOrGenerateSoftwareInstallerTitleID(ctx, payload.Title, payload.Source) + if err != nil { + return 0, err + } + + installScriptID, err := ds.getOrGenerateScriptContentsID(ctx, payload.InstallScript) + if err != nil { + return 0, err + } + + var postInstallScriptID *uint + if payload.PostInstallScript != "" { + sid, err := ds.getOrGenerateScriptContentsID(ctx, payload.PostInstallScript) + if err != nil { + return 0, err + } + postInstallScriptID = &sid + } + + var tid uint + if payload.TeamID != nil { + tid = *payload.TeamID + } + + stmt := ` +INSERT INTO software_installers ( + team_id, + global_or_team_id, + title_id, + storage_id, + filename, + version, + install_script_content_id, + pre_install_query, + post_install_script_content_id +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + + args := []interface{}{ + payload.TeamID, + tid, + titleID, + payload.StorageID, + payload.Filename, + payload.Version, + installScriptID, + payload.PreInstallQuery, + postInstallScriptID, + } + + res, err := ds.writer(ctx).ExecContext(ctx, stmt, args...) + if err != nil { + if isDuplicate(err) { + // already exists for this team/no team + err = alreadyExists("SoftwareInstaller", payload.Title) + } + return 0, ctxerr.Wrap(ctx, err, "insert software installer") + } + + id, _ := res.LastInsertId() + + return uint(id), nil +} + +func (ds *Datastore) getOrGenerateSoftwareInstallerTitleID(ctx context.Context, name, source string) (uint, error) { + titleID, err := ds.optimisticGetOrInsert(ctx, + ¶meterizedStmt{ + Statement: `SELECT id FROM software_titles WHERE name = ? AND source = ? AND browser = ''`, + Args: []interface{}{name, source}, + }, + ¶meterizedStmt{ + Statement: `INSERT INTO software_titles (name, source, browser) VALUES (?, ?, ?)`, + Args: []interface{}{name, source, ""}, + }, + ) + if err != nil { + return 0, err + } + + return titleID, nil +} + +func (ds *Datastore) GetSoftwareInstallerMetadata(ctx context.Context, id uint) (*fleet.SoftwareInstaller, error) { + query := ` +SELECT + id, + team_id, + title_id, + storage_id, + filename, + version, + install_script_content_id, + pre_install_query, + post_install_script_content_id, + uploaded_at +FROM + software_installers +WHERE + id = ?` + + var dest fleet.SoftwareInstaller + err := sqlx.GetContext(ctx, ds.reader(ctx), &dest, query, id) + if err != nil { + if err == sql.ErrNoRows { + return nil, ctxerr.Wrap(ctx, notFound("SoftwareInstaller").WithID(id), "get software installer metadata") + } + return nil, ctxerr.Wrap(ctx, err, "get software installer metadata") + } + + return &dest, nil +} + +func (ds *Datastore) DeleteSoftwareInstaller(ctx context.Context, id uint) error { + res, err := ds.writer(ctx).ExecContext(ctx, `DELETE FROM software_installers WHERE id = ?`, id) + if err != nil { + return ctxerr.Wrap(ctx, err, "delete software installer") + } + + rows, _ := res.RowsAffected() + if rows == 0 { + return notFound("SoftwareInstaller").WithID(id) + } + + return nil +} diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index c171fc1cdc..23d1406c3c 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1455,6 +1455,19 @@ type Datastore interface { // Apple hosts. It is optimized to update using only the information // available in the Apple MDM protocol. UpdateHostLockWipeStatusFromAppleMDMResult(ctx context.Context, hostUUID, cmdUUID, requestType string, succeeded bool) error + + /////////////////////////////////////////////////////////////////////////////// + // Software installers + // + + // MatchOrCreateSoftwareInstaller matches or creates a new software installer. + MatchOrCreateSoftwareInstaller(ctx context.Context, payload *UploadSoftwareInstallerPayload) (uint, error) + + // GetSoftwareInstallerMetadata returns the software installer corresponding to the id. + GetSoftwareInstallerMetadata(ctx context.Context, id uint) (*SoftwareInstaller, error) + + // DeleteSoftwareInstaller deletes the software installer corresponding to the id. + DeleteSoftwareInstaller(ctx context.Context, id uint) error } // MDMAppleStore wraps nanomdm's storage and adds methods to deal with diff --git a/server/fleet/service.go b/server/fleet/service.go index 8cf2779191..e6188fe395 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -996,4 +996,11 @@ type Service interface { LockHost(ctx context.Context, hostID uint) error UnlockHost(ctx context.Context, hostID uint) (unlockPIN string, err error) WipeHost(ctx context.Context, hostID uint) error + + /////////////////////////////////////////////////////////////////////////////// + // Software installers + // + + UploadSoftwareInstaller(ctx context.Context, payload *UploadSoftwareInstallerPayload) error + DeleteSoftwareInstaller(ctx context.Context, id uint) error } diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index bdc921dd20..ea24322a92 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -3,7 +3,9 @@ package fleet import ( "context" "errors" + "fmt" "io" + "path/filepath" "time" ) @@ -39,20 +41,28 @@ type SoftwareInstaller struct { // TeamID is the ID of the team. A value of nil means it is scoped to hosts that are assigned to // no team. TeamID *uint `json:"team_id" db:"team_id"` + // TitleID is the id of the software title associated with the software installer. + TitleID uint `json:"-" db:"title_id"` // Name is the name of the software package. - Name string `json:"name" db:"name"` + Name string `json:"name" db:"filename"` // Version is the version of the software package. Version string `json:"version" db:"version"` // UploadedAt is the time the software package was uploaded. - UploadedAt string `json:"uploaded_at" db:"uploaded_at"` + UploadedAt time.Time `json:"uploaded_at" db:"uploaded_at"` // InstallerID is the unique identifier for the software package metadata in Fleet. - InstallerID uint `json:"-" db:"installer_id"` + InstallerID uint `json:"-" db:"id"` // InstallScript is the script to run to install the software package. - InstallScript string `json:"install_script" db:"install_script"` + InstallScript string `json:"install_script" db:"-"` + // InstallScriptContentID is the ID of the install script content. + InstallScriptContentID uint `json:"-" db:"install_script_content_id"` // PreInstallQuery is the query to run as a condition to installing the software package. - PreInstallQuery string `json:"pre_install_query" db:"pre_install_condition"` + PreInstallQuery string `json:"pre_install_query" db:"pre_install_query"` // PostInstallScript is the script to run after installing the software package. - PostInstallScript string `json:"post_install_script"` + PostInstallScript string `json:"post_install_script" db:"-"` + // PostInstallScriptContentID is the ID of the post-install script content. + PostInstallScriptContentID *uint `json:"-" db:"post_install_script_content_id"` + // StorageID is the unique identifier for the software package in the software installer store. + StorageID string `json:"-" db:"storage_id"` } // AuthzType implements authz.AuthzTyper. @@ -105,6 +115,32 @@ type HostSoftwareInstallerResult struct { PostInstallScriptOutput string `json:"post_install_script_output" db:"post_install_script_output"` } +type UploadSoftwareInstallerPayload struct { + TeamID *uint + InstallScript string + PreInstallQuery string + PostInstallScript string + InstallerFile io.ReadSeeker // TODO: maybe pull this out of the payload and only pass it to methods that need it (e.g., won't be needed when storing metadata in the database) + StorageID string + Filename string + Title string + Version string + Source string +} + +func SofwareInstallerSourceFromFilename(filename string) (string, error) { + switch ext := filepath.Ext(filename); ext { + case ".deb": + return "deb_packages", nil + case ".exe", ".msi": + return "programs", nil + case ".pkg": + return "pkg_packages", nil + default: + return "", fmt.Errorf("unsupported file type: %s", filename) + } +} + // HostSoftwareWithInstaller represents the list of software installed on a // host with installer information if a matching installer exists. This is the // payload returned by the "Get host's (device's) software" endpoints. diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index d415ffecfa..a4d3b9253c 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -921,6 +921,12 @@ type WipeHostViaWindowsMDMFunc func(ctx context.Context, host *fleet.Host, cmd * type UpdateHostLockWipeStatusFromAppleMDMResultFunc func(ctx context.Context, hostUUID string, cmdUUID string, requestType string, succeeded bool) error +type MatchOrCreateSoftwareInstallerFunc func(ctx context.Context, payload *fleet.UploadSoftwareInstallerPayload) (uint, error) + +type GetSoftwareInstallerMetadataFunc func(ctx context.Context, id uint) (*fleet.SoftwareInstaller, error) + +type DeleteSoftwareInstallerFunc func(ctx context.Context, id uint) error + type DataStore struct { HealthCheckFunc HealthCheckFunc HealthCheckFuncInvoked bool @@ -2275,6 +2281,15 @@ type DataStore struct { UpdateHostLockWipeStatusFromAppleMDMResultFunc UpdateHostLockWipeStatusFromAppleMDMResultFunc UpdateHostLockWipeStatusFromAppleMDMResultFuncInvoked bool + MatchOrCreateSoftwareInstallerFunc MatchOrCreateSoftwareInstallerFunc + MatchOrCreateSoftwareInstallerFuncInvoked bool + + GetSoftwareInstallerMetadataFunc GetSoftwareInstallerMetadataFunc + GetSoftwareInstallerMetadataFuncInvoked bool + + DeleteSoftwareInstallerFunc DeleteSoftwareInstallerFunc + DeleteSoftwareInstallerFuncInvoked bool + mu sync.Mutex } @@ -5434,3 +5449,24 @@ func (s *DataStore) UpdateHostLockWipeStatusFromAppleMDMResult(ctx context.Conte s.mu.Unlock() return s.UpdateHostLockWipeStatusFromAppleMDMResultFunc(ctx, hostUUID, cmdUUID, requestType, succeeded) } + +func (s *DataStore) MatchOrCreateSoftwareInstaller(ctx context.Context, payload *fleet.UploadSoftwareInstallerPayload) (uint, error) { + s.mu.Lock() + s.MatchOrCreateSoftwareInstallerFuncInvoked = true + s.mu.Unlock() + return s.MatchOrCreateSoftwareInstallerFunc(ctx, payload) +} + +func (s *DataStore) GetSoftwareInstallerMetadata(ctx context.Context, id uint) (*fleet.SoftwareInstaller, error) { + s.mu.Lock() + s.GetSoftwareInstallerMetadataFuncInvoked = true + s.mu.Unlock() + return s.GetSoftwareInstallerMetadataFunc(ctx, id) +} + +func (s *DataStore) DeleteSoftwareInstaller(ctx context.Context, id uint) error { + s.mu.Lock() + s.DeleteSoftwareInstallerFuncInvoked = true + s.mu.Unlock() + return s.DeleteSoftwareInstallerFunc(ctx, id) +} diff --git a/server/service/handler.go b/server/service/handler.go index 3a5fb87051..ba922791be 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -371,6 +371,10 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC ue.GET("/api/_version_/fleet/software/titles", listSoftwareTitlesEndpoint, listSoftwareTitlesRequest{}) ue.GET("/api/_version_/fleet/software/titles/{id:[0-9]+}", getSoftwareTitleEndpoint, getSoftwareTitleRequest{}) + // Sofware installers + ue.POST("/api/_version_/fleet/software/package", uploadSoftwareInstallerEndpoint, uploadSoftwareInstallerRequest{}) + ue.DELETE("/api/_version_/fleet/software/package/{id:[0-9]+}", deleteSoftwareInstallerEndpoint, deleteSoftwareInstallerRequest{}) + // Vulnerabilities ue.GET("/api/_version_/fleet/vulnerabilities", listVulnerabilitiesEndpoint, listVulnerabilitiesRequest{}) ue.GET("/api/_version_/fleet/vulnerabilities/{cve}", getVulnerabilityEndpoint, getVulnerabilityRequest{}) diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 7dc1ee4fce..40cf1e70ff 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -362,6 +362,12 @@ func (s *integrationMDMTestSuite) TearDownTest() { _, err := tx.ExecContext(ctx, "DELETE FROM host_mdm_apple_declarations") return err }) + + // clear any lingering software installers + mysql.ExecAdhocSQL(t, s.ds, func(tx sqlx.ExtContext) error { + _, err := tx.ExecContext(ctx, "DELETE FROM software_installers") + return err + }) } func (s *integrationMDMTestSuite) mockDEPResponse(handler http.Handler) { @@ -8459,3 +8465,165 @@ func (s *integrationMDMTestSuite) TestIsServerBitlockerStatus() { require.NotNil(t, hr.Host.MDM.OSSettings.DiskEncryption.Status) require.Equal(t, fleet.DiskEncryptionEnforcing, *hr.Host.MDM.OSSettings.DiskEncryption.Status) } + +func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadAndDelete() { + t := s.T() + + openFile := func(name string) *os.File { + f, err := os.Open(filepath.Join("testdata", "software-installers", name)) + require.NoError(t, err) + return f + } + + uploadSoftwareInstaller := func(payload *fleet.UploadSoftwareInstallerPayload, expectedStatus int, expectedError string) { + f := openFile(payload.Filename) + defer f.Close() + + payload.InstallerFile = f + + var b bytes.Buffer + w := multipart.NewWriter(&b) + + // add the software field + fw, err := w.CreateFormFile("software", payload.Filename) + require.NoError(t, err) + n, err := io.Copy(fw, payload.InstallerFile) + require.NoError(t, err) + require.NotZero(t, n) + + // add the team_id field + if payload.TeamID != nil { + require.NoError(t, w.WriteField("team_id", fmt.Sprintf("%d", *payload.TeamID))) + } + // add the remaining fields + require.NoError(t, w.WriteField("install_script", payload.InstallScript)) + require.NoError(t, w.WriteField("pre_install_query", payload.PreInstallQuery)) + require.NoError(t, w.WriteField("post_install_script", payload.PostInstallScript)) + + w.Close() + + headers := map[string]string{ + "Content-Type": w.FormDataContentType(), + "Accept": "application/json", + "Authorization": fmt.Sprintf("Bearer %s", s.token), + } + + r := s.DoRawWithHeaders("POST", "/api/latest/fleet/software/package", b.Bytes(), expectedStatus, headers) + if expectedError != "" { + errMsg := extractServerErrorText(r.Body) + require.Contains(t, errMsg, expectedError) + } + } + + checkSoftwareTitle := func(t *testing.T, title string, source string) uint { + var id uint + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(context.Background(), q, &id, `SELECT id FROM software_titles WHERE name = ? AND source = ? AND browser = ''`, title, source) + }) + return id + } + + checkScriptContentsID := func(t *testing.T, id uint, expectedContents string) { + var contents string + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(context.Background(), q, &contents, `SELECT contents FROM script_contents WHERE id = ?`, id) + }) + require.Equal(t, expectedContents, contents) + } + + checkSoftwareInstaller := func(t *testing.T, payload *fleet.UploadSoftwareInstallerPayload) uint { + var id uint + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + var tid uint + if payload.TeamID != nil { + tid = *payload.TeamID + } + return sqlx.GetContext(context.Background(), q, &id, `SELECT id FROM software_installers WHERE global_or_team_id = ? AND filename = ?`, tid, payload.Filename) + }) + require.NotZero(t, id) + + meta, err := s.ds.GetSoftwareInstallerMetadata(context.Background(), id) + require.NoError(t, err) + + if payload.TeamID != nil { + require.Equal(t, *payload.TeamID, *meta.TeamID) + } else { + require.Nil(t, meta.TeamID) + } + + checkScriptContentsID(t, meta.InstallScriptContentID, payload.InstallScript) + + if payload.PostInstallScript != "" { + require.NotNil(t, meta.PostInstallScriptContentID) + checkScriptContentsID(t, *meta.PostInstallScriptContentID, payload.PostInstallScript) + } else { + require.Nil(t, meta.PostInstallScriptContentID) + } + + require.Equal(t, payload.PreInstallQuery, meta.PreInstallQuery) + require.Equal(t, payload.StorageID, meta.StorageID) + require.Equal(t, payload.Filename, meta.Name) + require.Equal(t, payload.Version, meta.Version) + require.Equal(t, checkSoftwareTitle(t, payload.Title, "deb_packages"), meta.TitleID) + require.NotZero(t, meta.UploadedAt) + + return meta.InstallerID + } + + t.Run("upload no team software installer", func(t *testing.T) { + payload := &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "some install script", + PreInstallQuery: "some pre install query", + PostInstallScript: "some post install script", + Filename: "ruby.deb", + // additional fields below are pre-populated so we can re-use the payload later for the test assertions + Title: "ruby", + Version: "1:2.5.1", + Source: "deb_packages", + StorageID: "df06d9ce9e2090d9cb2e8cd1f4d7754a803dc452bf93e3204e3acd3b95508628", + } + + uploadSoftwareInstaller(payload, http.StatusOK, "") + + // check the software installer + installerID := checkSoftwareInstaller(t, payload) + + // upload again fails + uploadSoftwareInstaller(payload, http.StatusConflict, "already exists") + + // delete the installer + s.Do("DELETE", fmt.Sprintf("/api/latest/fleet/software/package/%d", installerID), nil, http.StatusNoContent) + }) + + t.Run("create team software installer", func(t *testing.T) { + var createTeamResp teamResponse + s.DoJSON("POST", "/api/latest/fleet/teams", &fleet.Team{ + Name: t.Name(), + }, http.StatusOK, &createTeamResp) + require.NotZero(t, createTeamResp.Team.ID) + + payload := &fleet.UploadSoftwareInstallerPayload{ + TeamID: &createTeamResp.Team.ID, + InstallScript: "another install script", + PreInstallQuery: "another pre install query", + PostInstallScript: "another post install script", + Filename: "ruby.deb", + // additional fields below are pre-populated so we can re-use the payload later for the test assertions + Title: "ruby", + Version: "1:2.5.1", + Source: "deb_packages", + StorageID: "df06d9ce9e2090d9cb2e8cd1f4d7754a803dc452bf93e3204e3acd3b95508628", + } + + uploadSoftwareInstaller(payload, http.StatusOK, "") + + // check the software installer + installerID := checkSoftwareInstaller(t, payload) + + // upload again fails + uploadSoftwareInstaller(payload, http.StatusConflict, "already exists") + + // delete the installer + s.Do("DELETE", fmt.Sprintf("/api/latest/fleet/software/package/%d", installerID), nil, http.StatusNoContent) + }) +} diff --git a/server/service/software_installers.go b/server/service/software_installers.go new file mode 100644 index 0000000000..98ce9cd583 --- /dev/null +++ b/server/service/software_installers.go @@ -0,0 +1,142 @@ +package service + +import ( + "context" + "fmt" + "mime/multipart" + "net/http" + "strconv" + + "github.com/docker/go-units" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" +) + +type uploadSoftwareInstallerRequest struct { + File *multipart.FileHeader + TeamID *uint + InstallScript string + PreInstallQuery string + PostInstallScript string +} + +type uploadSoftwareInstallerResponse struct { + Err error `json:"error,omitempty"` +} + +// TODO: We parse the whole body before running svc.authz.Authorize. +// An authenticated but unauthorized user could abuse this. +func (uploadSoftwareInstallerRequest) DecodeRequest(ctx context.Context, r *http.Request) (interface{}, error) { + decoded := uploadSoftwareInstallerRequest{} + err := r.ParseMultipartForm(512 * units.MiB) + if err != nil { + return nil, &fleet.BadRequestError{ + Message: "failed to parse multipart form", + InternalErr: err, + } + } + + if r.MultipartForm.File["software"] == nil || len(r.MultipartForm.File["software"]) == 0 { + return nil, &fleet.BadRequestError{ + Message: "software multipart field is required", + InternalErr: err, + } + } + + decoded.File = r.MultipartForm.File["software"][0] + + if decoded.File.Size > 500*units.MiB { + // TODO: Should we try to assess the size earlier in the request processing (before parsing the form)? + return nil, &fleet.BadRequestError{ + Message: "The maximum file size is 500 MB.", + } + } + + // default is no team + val, ok := r.MultipartForm.Value["team_id"] + if ok && len(val) > 0 { + teamID, err := strconv.Atoi(val[0]) + if err != nil { + return nil, &fleet.BadRequestError{Message: fmt.Sprintf("failed to decode team_id in multipart form: %s", err.Error())} + } + decoded.TeamID = ptr.Uint(uint(teamID)) + } + + val, ok = r.MultipartForm.Value["install_script"] + if ok && len(val) > 0 { + decoded.InstallScript = val[0] + } + + val, ok = r.MultipartForm.Value["pre_install_query"] + if ok && len(val) > 0 { + decoded.PreInstallQuery = val[0] + } + + val, ok = r.MultipartForm.Value["post_install_script"] + if ok && len(val) > 0 { + decoded.PostInstallScript = val[0] + } + + return &decoded, nil +} + +func (r uploadSoftwareInstallerResponse) error() error { return r.Err } + +func uploadSoftwareInstallerEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { + req := request.(*uploadSoftwareInstallerRequest) + ff, err := req.File.Open() + if err != nil { + return uploadSoftwareInstallerResponse{Err: err}, nil + } + defer ff.Close() + + payload := &fleet.UploadSoftwareInstallerPayload{ + TeamID: req.TeamID, + InstallScript: req.InstallScript, + PreInstallQuery: req.PreInstallQuery, + PostInstallScript: req.PostInstallScript, + InstallerFile: ff, + Filename: req.File.Filename, + } + + if err := svc.UploadSoftwareInstaller(ctx, payload); err != nil { + return uploadSoftwareInstallerResponse{Err: err}, nil + } + return &uploadSoftwareInstallerResponse{}, nil +} + +func (svc *Service) UploadSoftwareInstaller(ctx context.Context, payload *fleet.UploadSoftwareInstallerPayload) error { + // skipauth: No authorization check needed due to implementation returning + // only license error. + svc.authz.SkipAuthorization(ctx) + + return fleet.ErrMissingLicense +} + +type deleteSoftwareInstallerRequest struct { + ID uint `url:"id"` +} + +type deleteSoftwareInstallerResponse struct { + Err error `json:"error,omitempty"` +} + +func (r deleteSoftwareInstallerResponse) error() error { return r.Err } +func (r deleteSoftwareInstallerResponse) Status() int { return http.StatusNoContent } + +func deleteSoftwareInstallerEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { + req := request.(*deleteSoftwareInstallerRequest) + err := svc.DeleteSoftwareInstaller(ctx, req.ID) + if err != nil { + return deleteSoftwareInstallerResponse{Err: err}, nil + } + return deleteSoftwareInstallerResponse{}, nil +} + +func (svc *Service) DeleteSoftwareInstaller(ctx context.Context, id uint) error { + // skipauth: No authorization check needed due to implementation returning + // only license error. + svc.authz.SkipAuthorization(ctx) + + return fleet.ErrMissingLicense +} diff --git a/server/service/testdata/software-installers/ruby.deb b/server/service/testdata/software-installers/ruby.deb new file mode 100644 index 0000000000000000000000000000000000000000..b8ac63e0448a8b93ee5c2e30f3ce3abe20f2c21a GIT binary patch literal 11340 zcmbulQ*bX%6rdR=H@5BE*tVVgV%v6ZY}>YN+qP}n&i;32cWNJ|Y98iPS67|tI{noB zJ)H9q@fbK7ne#y#n;2Rc*wPwW*cv$gBP1kbWMyJx<6vWFV<#kJ_%Hwec18vUHWn5_ z!vEI)8~V^pbPUi&cDBxrcGh&x299*@9{>A#MrL-l|8apn^f$YQx+V}1*vU$h7+7C6 zw4VYIFgnox2;$z&{ymCca5Nnt445ulZ{XxUJSo0^XaiH^g&NEeMlsj-eQMb8)lw?` zBgjE}@Ky5zRBzWKp1_M9#jfK^u1UY0oAt^aMyA-FRs8hPrx2#h5f^)Y?I_@4pkN^O*b4nSF~DYaLL>b{apzp(UGV`OK!c4(^jZ= zdmevu!icn>?znRJR-$0|cHWaRv6Bu$^UO>>+AwfFiqQ1Ec@?&A*;k?zx!Y?_urxR( zi{24*8Dfi8hHx{(Lk?)KSQ}y^s@8Gzge!wI&=`&eho165=8ed)6ksisC0kNI!PN1? zdd1`rRP*(I>z%XM$KrhSuLZ2`b%zq*gP%R54TdX%jE__4R)3L4 z6Jgf0P;x?Ur06Yr@}@Uo&ZRwV z%qf8zO{TvlZ7}%^;Y;B^q~kC%ob{FHgvifp1EF?30QCyKqk6d>#&rSA0~)-HWll`l z^7cgYV3cUx06N6^Ajy<#Q!#ciiKq0l&1hdm2SRPCQSQtE9p|up69>IGRDU;6QdYKu z?C_tBI`XzWA)Xf>X`_O_ErxDvYL$v&;&bE0j~Jn$zFSdl)FCd(LB<#sqtggVg@i@J zxJc!x;}XK4QM*t9{`Qmso}Fhi1O%=egAbkDuowvJ=zns-t_w7Ey<*{<;f~M--C`RH zVJTt}^L1S9>&Q>NP!btUqnLui*juu30oIUxF>d9r**aeM`d<#P0ZQY~&v}kY+MObV z_F$*=T6ZN7AL5^%KQHq=^3Ljc9~=-3>4VoN<++Isfa>(g$Kd_&q8os(Y2X$*j_QLz~)yPnb^tBIy&*0|3Pu8 zvfC{p6EQ?8cQS3TZ1Jek`C&ZUcPN`o9Fm?-ikc<38;StHta#owchJF4naECvqu5SThhZd+T} zoULWAZ6R}=Nh{ZmSdsRz??yNbOYJxin{4d=i;})>Lz;Ks7?#nXHDGO_jA4y!Orzkg~`G7 z3Acy(%noOD5<6f#HfS`%z(~qjzN#a3< zl`@wLg?~yn8cUSuw#TwI2su$nl~a<(#anp-n`~~a1WLse6u0rZz0RV4qAR_tA)rP< zCcc=w)U@C#Q*j!_utR_VUx$DaAsZe4W65NZYV>KUob;3xu)DUD(ZJvYppt54j3srm zcKe8bRnSNZ8f?AwQ^qat-X>Z94Gd_WU+y7%H5Rer5Fl+sg*(b>VeF~8zZ@IS%)A9VkBZ0BTWXZl~W`BK{N4)}k<8KijXd(Z8c?C}49%}u+`kkZR;F=mXx z>Gt)3<}sL)kL9gh4UcB_Fab1}=}!xe+443~>8?`8T-b7>St%pNk=S9nwz+s7HM7vAo1L5Cl#F$U>~sVfB-xzprT=Thik zjaHAHHq4}pcpOg~;b(Jbn6`1<&4)9;O43dfA|82V6)qK|hQTj_qJXP;CUgSZY~70H zB{Yj+mal!V5VGTlHS&u-c+O3AI<8YnK%SL3D3RzD2oP2BFtmY8wfK zXSJTe0$^H+aT?O>lEtoU8v@T!VxY}+hu@{~J97IV4(kSmjM_RqnH|xlN+u&E5MRuOXvsvC+Tv+)PR*=nM;YrS`BJkT37IDbmegiTJ1RZJ))gec2G9D zh>mXh@|ocg6aV~HQd%BnQjU|tqzkQ!GwL>sMA+RfH=xLEbB!r1Qz-kL~|C)?3=t0j9YBaX$kDz7JkB?xiG;c zfAf{GR_;yQ_|gYT*u8&HK^PEhm4?`TBYn?o&j&Pv>szFvx_^ISuIR>mBKXY}l~AN0 zqKXuJB*1uk`^Pz9eQH_H#=B|#K!J*MeKXn3jesj0|NMv?B_B1&p?A2@=&D-E#O?}f z#OX&hl!rynNsfjIKA`C7?`KZ(JV6*eCMQHJpG!F}1#nFrPmU%L%$y=43#+_wdCp|wg<`VZp(*;Hu>EdZs( z7M9F`;95sIoA>_=364T>wbpc|Tqchey|9;}1c@Y0N;#<8l;LhfOm3nCo*vl=di!i82Nm$0|0mM@(+yp)1C+9{Kxp?TROR< z<|dO(^mez5-9JkToHjUJOrZIhy_Qyz`{4u**X47>6@$qJYi`Lb2^KGnh1Lx z766Y;yH&kBY>~g*r zmvp(uSL5=jUkfM(qhPxsSt%N7my8p9g$peh`-|LtRFY}gLhS#h^BGcmZs{fMrk`g;y@i&|cPCjn-dmx$GV;s{>%X+OHE^c{oXY zH14LM7&j!yfKgo+ zB$vUV&qI~%sOdS-r`CVlD8xTGx$1Q;o^$I^m%mB$OOuBPZUx9p-16V7Rr8ffS0pr2 z*LT-#Ik@|aQPYt@0K06aZeB!5_=mQ@)ERR)cGr69 zp|ZSmW}O7e6-3%gyGbGWVyW6aAoI&|F;!y&tfD@u!=Z%Q@PV=QaeP&+E%G53B$Ah- z^?jpMjR$`4M=%wa|NCIi^3lMr#&i09`0|ro2|y6N8nYzUHNs$00e3JFa?M(^4ubb?rsQs3VvZH_3@4r9M=tjdU6Gvm+v5ewFoSi$=P zQal;cFTZ|$$4f!^f`|JW{jJbuAgEmtLopsy{b-*n@erq&jEV28qP&hwWy}?n2O|B zsQY$g)`B~epk%$(Ke^tm8#z!jvYIhXutiGz9sSRfptg2 z)ksj#(FI=a28-rnsZ`D0T1iykE`E2^N*{2>@v~qNx_icu{xpZjS-Ix6=9xG?Y z7QhL3%c6B_yyfztv@Tjo9VSaB$qIP)ahZ9GKZ*Nciz^Nxt^JXTj~qX-0Fq_DPm)0l zEip&^N7jZ%u9CY&q+O`7IIbCMMeaYxNL%!@M{ALRUsoSBqvjn+w zC}1E4jSp1zJ8=2KS_At>4rLzGBH*+(0UIE0ATv&HLTH9j(N{n$wUHH#K;7~Twp`zY zQB9uVKNyUsf3M?;n&8=;e~L?{5%j;M?k=RMnZ+%6*xGx#haZOtWd9`w*B#6+p7!30gAo38Yi;~eqq9VE+-f4p@YkI!aTH0BSbz^EyqXi>35T6;ez@SKr_?19%1 zULb$M8HJJHn017(Vo`Vo50`j1?N%Wp;$N_Up@BNS+T%axts4FNFgZCY0xu>jNv+lz zXuz|#arw*ECldETdRHwMegXj#hv@?E{BJHuSUg`w4;A+?{>xjMsP7D@Xc~<$=TE^o zoJCUtN@B_W=)OrQ?8kF#dnKCH2hE1gDJ zUe?$u^ODEWEHNW&sLrqEm|<&FAvV{2B%OUafXgm0m|M1BiGl-aShurXM5SonCSy>4 zrii6n&XBO$Mk1eCH)wYpN1JS9;u}}&hLgu~s640xrY7N};Hywv?@)5KQ1z}SB!p%z zfkN39E&9P`Vg0r3au)3PD7s|l8gYKo;O|e z@HX1B(R{BnXc6l84TjG9h4>?+G}9`mqPJRGBU6D(;e;|?;@UgwRUp<(ahmt&Lz|7F znwP{h!*~prne;)5_gr?y#CwB|bk9UsT;gK686D%80}=6}46rN8fo6z2zM`d>UIBUD zqD^}R3$tx=Jt)UQJF#1`U9C@uUtUQm0-|iM6P}K`MK|JU&t9r7)1KM9U7TC;;+2n} zsFU&4EY9RtQcA$lC?<>&VG~W`=K|qzj3_3GesUbrk{QHAsXguBA9s}8M5Dd4{Py?` zkh`~oFUSsQtcK+UOt39o4?|ZH-MOTb(2J*!coV=%mH)z%(ZXe#^Vb8DA0MIPqS`M0 zPH-Nz6jQEYIyGcNd;5w&G}Ft~{H@h=__u3JSa!q-Cg(B7Iy*zglJ22wPuv|sYQpXdo`OfSe^{Fa^60*d9rDGX<8V=KFu3cacW)7M2{t;su}+eQU0{9!FZ0RQT*AK5w~q#wM0!;w44VT5gb0WMa~*FL8K zC1X?H_jjwVgCKO72jHAgwbPl~=|!JDgX$-IENWqAJDJ8CY3s&hK#*FJh;=l?r!JjJ z;>IWU=Lf#73k@`qe2bd(Yo+&!gNJN^VH8SW+);+dl%DiH_0alLM{4|pf0R_U$?rgF z7r#7uD@CToPC{@> zi;3}xvg;+=ms})lGroYwsnIZan5|U-1(FHSF^c-^P?$5+KEsQiK72nU{5zSGa3GgB z+?_wBm%>cEIp{_nBYZG-E{W(ZxTx}4>lq7xRd}F_<2&*}TSc%QyB%a;KU*hFqc&9?p-83sKW~@{c7azoet0g)v1CE}fhf!2u^!eeg( zc%;m&;T%S%DvNC^W8$d)e2#r0F5pg!-+vU$VEAGzYO^qcU!b{A?G z{=gv*vo`NIBFPaMU=@V%+a5nAa5)@?G`}!G2=ci#uhE|SBkU{ z3P*5&liVeeD@>3_>O7H}9^FLQ7so_bTFQ^I zLmBNZysIzKr)E^4p?!124HlUOi&+vCOYH^kfgIzQr-DxjQ80hezt{A0+F0HX<<^}< z#=83U?f40z&IuO7?2Xg)vL+@d0@Ga(_$QgDDyb)z(E>%BE0X9nmbrO z5M#8+<0G({l(=%*CA2MGlN~|B-rK1m=Zf+k+iC&_VnbnZidE7U->O- zH_eB0c|I(FMG3B?b-Cf0wEM6>$gkOI^H^KW6Y`7kjhJI8T*up*0Z^k-Z-+(4n!QkF z<{-ymUVqsG=*%j!^punjhsw6!5W$G%1ADiJU9r7hV7W><{(zRoZN(wg9{)p~RC&*0 z&AVM=O&s$laPq))?z4vW6zUXFE1~avMNYv-EjIc*qBXS{mX#-BQej+T(?M&qnN)ps zLIVGyT8g5gch2V(CyP-xl{rBA*zx$YvjW(J<)YP`+PU}j

S z4G7A|sM~|u9P+U1>f{?}Un=$`!I(8jeF?E&ycu*-S9&07Dg?gIT>{7^@Mw<&WrGCg z2ls_6zCYXIWr|l>1)ZA+H@F(oMwA|hWg??pW^gi5Y9_XeavX}=AE3I%{{2-Xa|*MV zwqF#U8#Vl=t6V(~Y(9QmPuQbqIzgSGrO())4K`3O^&H#gb(d_iaedju6E+jR+z9D@ zfU<7((FGi)b2#?7ixo>BeVow=Vrho8Uinj<%w>pj(b&8(@G2(ZA(bAmX2X51H3OUW zsn`Bo@v3E5J6+}M!WzI*2TCiw7+k{(rkc26Mg1FGvuM`mQ_uP&L_nMn$1B#s!~p~j z;&?E$wZu|hSQ#q77ghUI;3%hhx+^u1V z5(Y(`z`7&^lv5yS_t@usC5Z?rA365|I?y)Dl5-cCc{g&GG`RTWCD6wCg+!5w6#Mm$ zv+fTIxWUv~S>8TjQgq%3i#EbD?ccD0;?@}fg3=5+j+zGGT7-%itIrH)6gI%EX zify@gN@;!9yXESjBUIUyCV^0W!-P^1@kB0qf8-aj1)dChu63ZXPxg ziTRwXP?W0x!O4(^$E@pA+Djylu|*u4&Jrldf>&^~Un3b`Z*HAzYOyqp|QxcN?Ve0I;0TbC+bq zEHK`6{M;JNy^Rzt5lUb#blN=camWs_ijF2{+4NafAm{z|9z_Qa%^HJ|6qm5tnUND; z7>cHxCx1O`;l4<+=Ubj$6jY^&CR-TsB(jv2QlATGF8}uRw z(Rj<3#L?XaJq`HMbTrNBgJjI;T4Eeee~m>ZcB1v0^Y8@ecmD});*9$Q;3t`~HH4^N zmuGCdkFEI&R>|d{;x3-H?7<5YxD$a3sQ4CcO1ioPssv|B=^TTz;LhmC3^a(z-eSPx z`JhB^Lv^lS%e>(C9@!8&&=%Z@ORAgQz614%>)R)6ZJ)SU!te+NXnR?WU9jPG2ZXt92D?0~rF-3WC~hXuyVKr5hUL*we4|)nEBCFM zY0xTy7~K$q09{>1$J7VUPY$=e<1H8ba zptfmSE{(Om`-w*UE7ozFg@I+Sf!}5Zh8mU9Utj(1RDK1e8Cv!ITf2A%aN z7_Sggt@pW(3X3Dat~5b+AgxeqW`@HZilIE3J|;PZ)+0 z(7ev0kzg)i!HPWJA~mpNmWzgX31!q`uZ#9aREbq3i4S+PWq?1}^_nxPM@p8G#yE_H5bq2kG6t=#I#u|0G1Z!oxcf?_ zIZz(|Yy_$-_yATlxNSmC^z;joJULgg8tz1&U$FJabj5A;_ugG@!36$OyZqF|?6Fej zW^@9&O&ZLh>_LIfw37IF-?D2ww7?&PI!LRpQkNR=F82F9Di(cUL*h$MCfV!|IsqXa z6TzF0wgdc&(uNXfU+Pl?0~Rgb9p^xmIP`m1m9G8~bwrBYT^&&f*e}RYV`Eu1S8Ohz z?OLG4)0N$}97DAR8iHMMhyF`Y+w)h#iwf7xEk-!i-@pY|FYs$7jk;m!&JDqFJ5l{1 zI(s2LJWdpMv17eE*hdBmgf}+u47GA__hrYM(P!3^$eO65bPaA#zkYbix2G2%sn*<< zHXulo(+5G|tI1AAzzFK>|bC&f&C7Ig6!_@yFi(aGX-)Dkl~EI2*#u3wSB2vyDfRC% zO(?ir$;d|jmMb^kNBJ0#p_hBPD68J$d=oz?413d5qa+M@XnlR8B>%Q+@#4?i6my;# z;S^<_0DM!R(3z1gEXBwMI$&B({n6t_FNZXK)$!b67ZbeuR)p3vkorF7t8|^qF2dN3 zCaqhBMqnr;rtdHsHlV5W(*tHnR`_mDDw;As8?nr~y_s;pEU#MBifwfPzN$%u|X2^h+Kzcp5-V`bvRcR@QF z;&B=Lr}-lKD{C6k^#RMbYDv&uCmi&uQL5@$_3Jmq9We_~E^w?~Y6NSO%1jGmzv3Pt zZ_hLVxt`LTYE2-4`uoXIj14#5pEa3oE~)C(!4nsA3`$u5W!Y7JFxS6;u!aZElqtRG zGXh3&X>bRouaC3<@u}vMdojrp@`7>T)d7};!_JsMu~;SHE5y$KkM&NGp?p;xNg%Z^ zatRrk$giDZAPCx)i5A4`>lwyZPZNqzRa^BvuUu788N$qmwhhx4jD*6d%}grPLV|F9 z7vtwP$qN?;ik|QYk*#-`ycHAXGH68NWqSF)(ep1rf~h3EGt#V|m{k3_-_ymgm}kTF zQzdkoWL>ULKH{II7j?%p%v(=Av-|F1CRp;5d(StzDW|6Lq!VK2nP+sZ{g#FOR#n(X zHXi_a-R0P#gnxQab9;Y$<{xJVPsijCZS>T_$j5lys(aX6Vcwi^WxXr#cod?YxC-Jp zV=y`F5pJPZ5qJ9rg&-TnR~eIEzwLq^gbp;YVeQZP9dgd-l3D?e!}FODOj;J{kljQT zm8wKfF#q13FAxUS&|TX;;Xf$o?PjJgy{9(WH7GPRJuv?ZU(2qxRm+Kt?8YEi4vtXJ zmh37e1{<5s_=$|jfA@cL2VS&9T2UW%h~*MCkn%m#za%i*(>M09m(WGKZ*4Ia9Smvm zm(Zqm|Gul!hPT^Dg(xT`VB^w+>|KI5WWk2JaBqG9FK;mrWZJdw2BT>wk9Idh!|Mo# z-H@W8)jN}=3QDcrIGpFU{i8wcEpkZPqUus7_F)=~Vm!k=vO(`7ao0)PNY0VU+r%BP zwDow8US%!1RI+TwDZULq2!cd5zMYVY(`ll+d|@MTo1eVN@WW0L=*L^Dhguv1q`)U! z7EK77gH{O#@{i3HOd=`i{*#^SWeKw+FEQ)LQ&WJM!WmEY64U~sh&UxF>i=f{id%HB zoRm7)Y`2d&rxraE2eKsAq(5;1Ijvp|YD@(KQBaxndpi1eqgzDzxT=^!Z10P`{%Ia> zzDpM>5h8<%+))Vziv==qW>JO_mUOi2;A||B!$Sz-K{rQ7fm9G3nN+E3>|7N_dC-K_K%;iO?{|+OeS=j?AN5O8?whuYJq=3_)@+ce)e3l@IUE?*>M}r zS^Z;Ph&PnAonPJ}K2l6M%OWhzAhv%PEpd-=Mp1+>#(4~27`Ns|;wFBYq#v&h9qta5 zJpV>XUaBCK1H#%GMzeW5OEsY|mr};BC%w}c^4p339Du!HHN^{SC}Yhg6PksnTN6Q$ z04y|L&1j75O9MbdOOf?;Dyn0_BQ$Wf$uk-nviAxe1%IT~V3yhAx*uzoum@E; zIpRJuKWMQvKt@Mu3?pr>7_>AiWv7tJaL>BVC4?8M7wDpUns%HHe8Mi2GqtODF_L?z z4VYXp_*#X&s%5p4j zdi=~)&B+rBrrU|})J>uUQ4rW6@vJx0?!0>VAGC|)a1|AJ7}sF(lKesSzm)Uc+`3y0We6wz4 zr;>eNS5|Wm&wFJBcU_Xy#_p(haS>%^A5845=wXtvAQD#s9ZkaEGUGk0aRx&6WnvRw zU0=TTmdxJYxJ26K;#v5A0l!_O_AK7vr1Y!Eni3GttIx9C28_nSm240|+3~*f90G9D zbXN)N?76E44{LkKGG7>ao_7-|mvG2*!ylnU%tlS8fnMH$#!DR;+SW9nA(1i95#*L4 zQxl^XHHn1W?Y!?Er@set(^?K2=7a|fRg){CD;_hQpn>#I9CH$lgX+007k$` zsXH5!E9PfhBQQPeBsP<|5b+h^gGA56ZF-#LWvL{Fu5?$uKVD(j@R3bC34q^LuN*`o zA&bu-bst@X1a@nblzx5N|F6t%2``xUEHYkpn5w&?WfWPu(AMumk zVv5n$gt{052Cu$HouqkH9$QzJjxV9Z?wZ39CMR5S<=b!<6Q9d1D7#=(GyTEjoH0u8 t0u4NXm80$>O&$}3z<(;)G``h9=Tj&!5OAo>e;yc+UcCwK|KBtFzW{Xu={*1d literal 0 HcmV?d00001 From 123fdc72b016dd26cab567e0079653336fc702f3 Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Thu, 2 May 2024 18:00:06 -0300 Subject: [PATCH 12/56] add endpoint to send software installer requests (#18711) #18334 # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements) - [x] Added/updated tests - [x] Manual QA for all new/changed functionality --- ee/server/service/software_installers.go | 46 +++++ pkg/file/msi.go | 4 +- server/authz/policy.rego | 22 +++ server/authz/policy_test.go | 57 +++++++ server/datastore/mysql/software_installers.go | 50 ++++++ .../mysql/software_installers_test.go | 78 +++++++++ server/fleet/datastore.go | 3 + server/fleet/service.go | 3 + server/fleet/software_installer.go | 9 + server/mock/datastore_mock.go | 12 ++ server/service/handler.go | 1 + server/service/integration_mdm_test.go | 158 ++++++++++++------ server/service/software_installers.go | 36 ++++ 13 files changed, 427 insertions(+), 52 deletions(-) create mode 100644 server/datastore/mysql/software_installers_test.go diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index d70bda7f40..dd038ef9a6 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -3,6 +3,8 @@ package service import ( "context" "encoding/hex" + "errors" + "net/http" "path/filepath" "strings" @@ -108,3 +110,47 @@ func (svc *Service) DeleteSoftwareInstaller(ctx context.Context, id uint) error return nil } + +func (svc *Service) InstallSoftwareTitle(ctx context.Context, hostID uint, softwareTitleID uint) error { + // we need to use ds.Host because ds.HostLite doesn't return the orbit + // node key + host, err := svc.ds.Host(ctx, hostID) + if err != nil { + // if error is because the host does not exist, check first if the user + // had access to install software (to prevent leaking valid host ids). + if fleet.IsNotFound(err) { + if err := svc.authz.Authorize(ctx, &fleet.HostSoftwareInstallerResultAuthz{}, fleet.ActionWrite); err != nil { + return err + } + } + svc.authz.SkipAuthorization(ctx) + return ctxerr.Wrap(ctx, err, "get host") + } + + if host.OrbitNodeKey == nil || *host.OrbitNodeKey == "" { + // fleetd is required to install software so if the host is + // enrolled via plain osquery we return an error + svc.authz.SkipAuthorization(ctx) + // TODO(roberto): for cleanup task, confirm with product error message. + return fleet.NewUserMessageError(errors.New("Host doesn't have fleetd installed"), http.StatusUnprocessableEntity) + } + + // authorize with the host's team + if err := svc.authz.Authorize(ctx, &fleet.HostSoftwareInstallerResultAuthz{HostTeamID: host.TeamID}, fleet.ActionWrite); err != nil { + return err + } + + err = svc.ds.InsertSoftwareInstallRequest(ctx, hostID, softwareTitleID, host.TeamID) + if err != nil { + if fleet.IsNotFound(err) { + return &fleet.BadRequestError{ + Message: "The software title provided doesn't have an installer", + InternalErr: ctxerr.Wrapf(ctx, err, "couldn't find an installer for software title"), + } + } + + return ctxerr.Wrap(ctx, err, "inserting software install request") + } + + return nil +} diff --git a/pkg/file/msi.go b/pkg/file/msi.go index 270e5242e4..118eec922e 100644 --- a/pkg/file/msi.go +++ b/pkg/file/msi.go @@ -270,7 +270,7 @@ func msiDecodeRune(x rune) rune { return x - 10 - 26 + 'a' } else if x == 10+26+26 { return '.' - } else { - return '_' } + + return '_' } diff --git a/server/authz/policy.rego b/server/authz/policy.rego index 8e46820688..9d74fb19d4 100644 --- a/server/authz/policy.rego +++ b/server/authz/policy.rego @@ -673,6 +673,28 @@ allow { action == write } +## +# Host software installs +## + +# Global admins and maintainers can write (install) software on hosts (not +# gitops as this is not something that relates to fleetctl apply). +allow { + object.type == "host_software_installer_result" + subject.global_role == [admin, maintainer][_] + action == write +} + +# Team admin and maintainers can write (install) software on hosts for their +# teams (not gitops as this is not something that relates to fleetctl apply). +allow { + object.type == "host_software_installer_result" + not is_null(object.host_team_id) + team_role(subject, object.host_team_id) == [admin, maintainer][_] + action == write +} + + ## # Apple and Windows MDM ## diff --git a/server/authz/policy_test.go b/server/authz/policy_test.go index 0f2ce9b340..ba269e32c9 100644 --- a/server/authz/policy_test.go +++ b/server/authz/policy_test.go @@ -595,6 +595,63 @@ func TestAuthorizeSoftwareInstaller(t *testing.T) { }) } +func TestAuthorizeHostSoftwareInstallerResult(t *testing.T) { + t.Parallel() + + noTeamInstallRequest := &fleet.HostSoftwareInstallerResultAuthz{} + team1InstallRequest := &fleet.HostSoftwareInstallerResultAuthz{HostTeamID: ptr.Uint(1)} + team2InstallRequest := &fleet.HostSoftwareInstallerResultAuthz{HostTeamID: ptr.Uint(2)} + runTestCases(t, []authTestCase{ + {user: nil, object: noTeamInstallRequest, action: write, allow: false}, + {user: nil, object: team1InstallRequest, action: write, allow: false}, + {user: nil, object: team2InstallRequest, action: write, allow: false}, + + {user: test.UserNoRoles, object: noTeamInstallRequest, action: write, allow: false}, + {user: test.UserNoRoles, object: team1InstallRequest, action: write, allow: false}, + {user: test.UserNoRoles, object: team2InstallRequest, action: write, allow: false}, + + {user: test.UserAdmin, object: noTeamInstallRequest, action: write, allow: true}, + {user: test.UserAdmin, object: team1InstallRequest, action: write, allow: true}, + {user: test.UserAdmin, object: team2InstallRequest, action: write, allow: true}, + + {user: test.UserMaintainer, object: noTeamInstallRequest, action: write, allow: true}, + {user: test.UserMaintainer, object: team1InstallRequest, action: write, allow: true}, + {user: test.UserMaintainer, object: team2InstallRequest, action: write, allow: true}, + + {user: test.UserObserver, object: noTeamInstallRequest, action: write, allow: false}, + {user: test.UserObserver, object: team1InstallRequest, action: write, allow: false}, + {user: test.UserObserver, object: team2InstallRequest, action: write, allow: false}, + + {user: test.UserObserverPlus, object: noTeamInstallRequest, action: write, allow: false}, + {user: test.UserObserverPlus, object: team1InstallRequest, action: write, allow: false}, + {user: test.UserObserverPlus, object: team2InstallRequest, action: write, allow: false}, + + {user: test.UserGitOps, object: noTeamInstallRequest, action: write, allow: false}, + {user: test.UserGitOps, object: team1InstallRequest, action: write, allow: false}, + {user: test.UserGitOps, object: team2InstallRequest, action: write, allow: false}, + + {user: test.UserTeamGitOpsTeam1, object: noTeamInstallRequest, action: write, allow: false}, + {user: test.UserTeamGitOpsTeam1, object: team1InstallRequest, action: write, allow: false}, + {user: test.UserTeamGitOpsTeam1, object: team2InstallRequest, action: write, allow: false}, + + {user: test.UserTeamAdminTeam1, object: noTeamInstallRequest, action: write, allow: false}, + {user: test.UserTeamAdminTeam1, object: team1InstallRequest, action: write, allow: true}, + {user: test.UserTeamAdminTeam1, object: team2InstallRequest, action: write, allow: false}, + + {user: test.UserTeamMaintainerTeam1, object: noTeamInstallRequest, action: write, allow: false}, + {user: test.UserTeamMaintainerTeam1, object: team1InstallRequest, action: write, allow: true}, + {user: test.UserTeamMaintainerTeam1, object: team2InstallRequest, action: write, allow: false}, + + {user: test.UserTeamObserverTeam1, object: noTeamInstallRequest, action: write, allow: false}, + {user: test.UserTeamObserverTeam1, object: team1InstallRequest, action: write, allow: false}, + {user: test.UserTeamObserverTeam1, object: team2InstallRequest, action: write, allow: false}, + + {user: test.UserTeamObserverPlusTeam1, object: noTeamInstallRequest, action: write, allow: false}, + {user: test.UserTeamObserverPlusTeam1, object: team1InstallRequest, action: write, allow: false}, + {user: test.UserTeamObserverPlusTeam1, object: team2InstallRequest, action: write, allow: false}, + }) +} + func TestAuthorizeHost(t *testing.T) { t.Parallel() diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 1027c6fe11..afa4beb751 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -6,6 +6,7 @@ import ( "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/google/uuid" "github.com/jmoiron/sqlx" ) @@ -134,3 +135,52 @@ func (ds *Datastore) DeleteSoftwareInstaller(ctx context.Context, id uint) error return nil } + +func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareTitleID uint, teamID *uint) error { + var tmID uint + if teamID != nil { + tmID = *teamID + } + + const ( + insertStmt = ` + INSERT INTO host_software_installs + (execution_id, host_id, software_installer_id) + VALUES + (?, ?, ?) + ` + + getInstallerIDStmt = `SELECT id FROM software_installers WHERE title_id = ? AND global_or_team_id = ?` + + hostExistsStmt = `SELECT 1 FROM hosts WHERE id = ?` + ) + + // we need to explicitly do this check here because we can't set a FK constraint on the schema + var hostExists bool + err := sqlx.GetContext(ctx, ds.reader(ctx), &hostExists, hostExistsStmt, hostID) + if err != nil { + if err == sql.ErrNoRows { + return notFound("Host").WithID(hostID) + } + + return ctxerr.Wrap(ctx, err, "inserting new install software request") + } + + var installerID uint + err = sqlx.GetContext(ctx, ds.reader(ctx), &installerID, getInstallerIDStmt, softwareTitleID, tmID) + if err != nil { + if err == sql.ErrNoRows { + return notFound("SoftwareInstaller") + } + + return ctxerr.Wrap(ctx, err, "inserting new install software request") + } + + _, err = ds.writer(ctx).ExecContext(ctx, insertStmt, + hostID, + uuid.NewString(), + softwareTitleID, + ) + + return ctxerr.Wrap(ctx, err, "inserting new install software request") +} diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go new file mode 100644 index 0000000000..5645111406 --- /dev/null +++ b/server/datastore/mysql/software_installers_test.go @@ -0,0 +1,78 @@ +package mysql + +import ( + "context" + "testing" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func TestSoftwareInstallers(t *testing.T) { + ds := CreateMySQLDS(t) + + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"InsertSoftwareInstallRequest", testInsertSoftwareInstallRequest}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testInsertSoftwareInstallRequest(t *testing.T, ds *Datastore) { + ctx := context.Background() + + // create a team + team, err := ds.NewTeam(ctx, &fleet.Team{Name: "team 2"}) + require.NoError(t, err) + + cases := map[string]*uint{ + "no team": nil, + "team": &team.ID, + } + + for tc, teamID := range cases { + t.Run(tc, func(t *testing.T) { + // non-existent installer and host does the installer check first + err := ds.InsertSoftwareInstallRequest(ctx, 1, 1, teamID) + var nfe fleet.NotFoundError + require.ErrorAs(t, err, &nfe) + + // non-existent host + installerID, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + Title: "foo", + Source: "bar", + InstallScript: "echo", + TeamID: teamID, + }) + require.NoError(t, err) + installerMeta, err := ds.GetSoftwareInstallerMetadata(ctx, installerID) + require.NoError(t, err) + + err = ds.InsertSoftwareInstallRequest(ctx, 12, installerMeta.TitleID, teamID) + require.ErrorAs(t, err, &nfe) + + // successful insert + host, err := ds.NewHost(ctx, &fleet.Host{ + Hostname: "macos-test" + tc, + OsqueryHostID: ptr.String("osquery-macos" + tc), + NodeKey: ptr.String("node-key-macos" + tc), + UUID: uuid.NewString(), + Platform: "darwin", + TeamID: teamID, + }) + require.NoError(t, err) + err = ds.InsertSoftwareInstallRequest(ctx, host.ID, installerMeta.TitleID, teamID) + require.NoError(t, err) + }) + } +} diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 23d1406c3c..93792dcfee 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -490,6 +490,9 @@ type Datastore interface { ListSoftwareTitles(ctx context.Context, opt SoftwareTitleListOptions, tmFilter TeamFilter) ([]SoftwareTitle, int, *PaginationMetadata, error) SoftwareTitleByID(ctx context.Context, id uint, teamID *uint, tmFilter TeamFilter) (*SoftwareTitle, error) + // InsertSoftwareInstallRequest tracks a new request to install the provided software installer in the host + InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareInstallerID uint, teamID *uint) error + /////////////////////////////////////////////////////////////////////////////// // SoftwareStore diff --git a/server/fleet/service.go b/server/fleet/service.go index e6188fe395..3cbf73dd4f 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -625,6 +625,9 @@ type Service interface { ListSoftwareTitles(ctx context.Context, opt SoftwareTitleListOptions) ([]SoftwareTitle, int, *PaginationMetadata, error) SoftwareTitleByID(ctx context.Context, id uint, teamID *uint) (*SoftwareTitle, error) + // InstallSoftwareTitle installs a software title in the given host. + InstallSoftwareTitle(ctx context.Context, hostID uint, softwareTitleID uint) error + // ///////////////////////////////////////////////////////////////////////////// // Vulnerabilities diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index ea24322a92..844617b976 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -115,6 +115,15 @@ type HostSoftwareInstallerResult struct { PostInstallScriptOutput string `json:"post_install_script_output" db:"post_install_script_output"` } +type HostSoftwareInstallerResultAuthz struct { + HostTeamID *uint `json:"host_team_id"` +} + +// AuthzType implements authz.AuthzTyper. +func (s *HostSoftwareInstallerResultAuthz) AuthzType() string { + return "host_software_installer_result" +} + type UploadSoftwareInstallerPayload struct { TeamID *uint InstallScript string diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index a4d3b9253c..79b2686cf8 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -365,6 +365,8 @@ type ListSoftwareTitlesFunc func(ctx context.Context, opt fleet.SoftwareTitleLis type SoftwareTitleByIDFunc func(ctx context.Context, id uint, teamID *uint, tmFilter fleet.TeamFilter) (*fleet.SoftwareTitle, error) +type InsertSoftwareInstallRequestFunc func(ctx context.Context, hostID uint, softwareInstallerID uint, teamID *uint) error + type ListSoftwareForVulnDetectionFunc func(ctx context.Context, hostID uint) ([]fleet.Software, error) type ListSoftwareVulnerabilitiesByHostIDsSourceFunc func(ctx context.Context, hostIDs []uint, source fleet.VulnerabilitySource) (map[uint][]fleet.SoftwareVulnerability, error) @@ -1447,6 +1449,9 @@ type DataStore struct { SoftwareTitleByIDFunc SoftwareTitleByIDFunc SoftwareTitleByIDFuncInvoked bool + InsertSoftwareInstallRequestFunc InsertSoftwareInstallRequestFunc + InsertSoftwareInstallRequestFuncInvoked bool + ListSoftwareForVulnDetectionFunc ListSoftwareForVulnDetectionFunc ListSoftwareForVulnDetectionFuncInvoked bool @@ -3504,6 +3509,13 @@ func (s *DataStore) SoftwareTitleByID(ctx context.Context, id uint, teamID *uint return s.SoftwareTitleByIDFunc(ctx, id, teamID, tmFilter) } +func (s *DataStore) InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareInstallerID uint, teamID *uint) error { + s.mu.Lock() + s.InsertSoftwareInstallRequestFuncInvoked = true + s.mu.Unlock() + return s.InsertSoftwareInstallRequestFunc(ctx, hostID, softwareInstallerID, teamID) +} + func (s *DataStore) ListSoftwareForVulnDetection(ctx context.Context, hostID uint) ([]fleet.Software, error) { s.mu.Lock() s.ListSoftwareForVulnDetectionFuncInvoked = true diff --git a/server/service/handler.go b/server/service/handler.go index ba922791be..76c9d55e9d 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -370,6 +370,7 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC ue.GET("/api/_version_/fleet/software/titles", listSoftwareTitlesEndpoint, listSoftwareTitlesRequest{}) ue.GET("/api/_version_/fleet/software/titles/{id:[0-9]+}", getSoftwareTitleEndpoint, getSoftwareTitleRequest{}) + ue.POST("/api/v1/fleet/hosts/{host_id:[0-9]+}/software/install/{software_title_id:[0-9]+}", installSoftwareTitleEndpoint, installSoftwareRequest{}) // Sofware installers ue.POST("/api/_version_/fleet/software/package", uploadSoftwareInstallerEndpoint, uploadSoftwareInstallerRequest{}) diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 40cf1e70ff..230c5509b8 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -8469,52 +8469,6 @@ func (s *integrationMDMTestSuite) TestIsServerBitlockerStatus() { func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadAndDelete() { t := s.T() - openFile := func(name string) *os.File { - f, err := os.Open(filepath.Join("testdata", "software-installers", name)) - require.NoError(t, err) - return f - } - - uploadSoftwareInstaller := func(payload *fleet.UploadSoftwareInstallerPayload, expectedStatus int, expectedError string) { - f := openFile(payload.Filename) - defer f.Close() - - payload.InstallerFile = f - - var b bytes.Buffer - w := multipart.NewWriter(&b) - - // add the software field - fw, err := w.CreateFormFile("software", payload.Filename) - require.NoError(t, err) - n, err := io.Copy(fw, payload.InstallerFile) - require.NoError(t, err) - require.NotZero(t, n) - - // add the team_id field - if payload.TeamID != nil { - require.NoError(t, w.WriteField("team_id", fmt.Sprintf("%d", *payload.TeamID))) - } - // add the remaining fields - require.NoError(t, w.WriteField("install_script", payload.InstallScript)) - require.NoError(t, w.WriteField("pre_install_query", payload.PreInstallQuery)) - require.NoError(t, w.WriteField("post_install_script", payload.PostInstallScript)) - - w.Close() - - headers := map[string]string{ - "Content-Type": w.FormDataContentType(), - "Accept": "application/json", - "Authorization": fmt.Sprintf("Bearer %s", s.token), - } - - r := s.DoRawWithHeaders("POST", "/api/latest/fleet/software/package", b.Bytes(), expectedStatus, headers) - if expectedError != "" { - errMsg := extractServerErrorText(r.Body) - require.Contains(t, errMsg, expectedError) - } - } - checkSoftwareTitle := func(t *testing.T, title string, source string) uint { var id uint mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { @@ -8583,13 +8537,13 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadAndDelete() { StorageID: "df06d9ce9e2090d9cb2e8cd1f4d7754a803dc452bf93e3204e3acd3b95508628", } - uploadSoftwareInstaller(payload, http.StatusOK, "") + s.uploadSoftwareInstaller(payload, http.StatusOK, "") // check the software installer installerID := checkSoftwareInstaller(t, payload) // upload again fails - uploadSoftwareInstaller(payload, http.StatusConflict, "already exists") + s.uploadSoftwareInstaller(payload, http.StatusConflict, "already exists") // delete the installer s.Do("DELETE", fmt.Sprintf("/api/latest/fleet/software/package/%d", installerID), nil, http.StatusNoContent) @@ -8615,15 +8569,119 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadAndDelete() { StorageID: "df06d9ce9e2090d9cb2e8cd1f4d7754a803dc452bf93e3204e3acd3b95508628", } - uploadSoftwareInstaller(payload, http.StatusOK, "") + s.uploadSoftwareInstaller(payload, http.StatusOK, "") // check the software installer installerID := checkSoftwareInstaller(t, payload) // upload again fails - uploadSoftwareInstaller(payload, http.StatusConflict, "already exists") + s.uploadSoftwareInstaller(payload, http.StatusConflict, "already exists") // delete the installer s.Do("DELETE", fmt.Sprintf("/api/latest/fleet/software/package/%d", installerID), nil, http.StatusNoContent) }) } + +func (s *integrationMDMTestSuite) TestSoftwareInstallerNewInstallRequest() { + t := s.T() + + getTitleID := func(t *testing.T, title string, source string) uint { + var id uint + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(context.Background(), q, &id, `SELECT id FROM software_titles WHERE name = ? AND source = ? AND browser = ''`, title, source) + }) + return id + } + + var resp installSoftwareResponse + // non-existent host + s.DoJSON("POST", "/api/v1/fleet/hosts/1/software/install/1", nil, http.StatusNotFound, &resp) + + // create a host that doesn't have fleetd installed + h, err := s.ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now().Add(-1 * time.Minute), + OsqueryHostID: ptr.String(t.Name() + uuid.New().String()), + NodeKey: ptr.String(t.Name() + uuid.New().String()), + Hostname: fmt.Sprintf("%sfoo.local", t.Name()), + Platform: "darwin", + }) + require.NoError(t, err) + + // request fails + resp = installSoftwareResponse{} + s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/hosts/%d/software/install/1", h.ID), nil, http.StatusUnprocessableEntity, &resp) + + // host installs fleetd + setOrbitEnrollment(t, h, s.ds) + + // request fails because of non-existent title + resp = installSoftwareResponse{} + s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/hosts/%d/software/install/1", h.ID), nil, http.StatusBadRequest, &resp) + + payload := &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "another install script", + PreInstallQuery: "another pre install query", + PostInstallScript: "another post install script", + Filename: "ruby.deb", + Title: "ruby", + } + s.uploadSoftwareInstaller(payload, http.StatusOK, "") + + // install script request succeds + titleID := getTitleID(t, payload.Title, "deb_packages") + resp = installSoftwareResponse{} + s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/hosts/%d/software/install/%d", h.ID, titleID), nil, http.StatusAccepted, &resp) + + // TODO(roberto): once we have endpoints to retrieve installers, + // request them using the orbit node key +} + +func (s *integrationMDMTestSuite) uploadSoftwareInstaller(payload *fleet.UploadSoftwareInstallerPayload, expectedStatus int, expectedError string) { + t := s.T() + openFile := func(name string) *os.File { + f, err := os.Open(filepath.Join("testdata", "software-installers", name)) + require.NoError(t, err) + return f + } + + f := openFile(payload.Filename) + defer f.Close() + + payload.InstallerFile = f + + var b bytes.Buffer + w := multipart.NewWriter(&b) + + // add the software field + fw, err := w.CreateFormFile("software", payload.Filename) + require.NoError(t, err) + n, err := io.Copy(fw, payload.InstallerFile) + require.NoError(t, err) + require.NotZero(t, n) + + // add the team_id field + if payload.TeamID != nil { + require.NoError(t, w.WriteField("team_id", fmt.Sprintf("%d", *payload.TeamID))) + } + // add the remaining fields + require.NoError(t, w.WriteField("install_script", payload.InstallScript)) + require.NoError(t, w.WriteField("pre_install_query", payload.PreInstallQuery)) + require.NoError(t, w.WriteField("post_install_script", payload.PostInstallScript)) + + w.Close() + + headers := map[string]string{ + "Content-Type": w.FormDataContentType(), + "Accept": "application/json", + "Authorization": fmt.Sprintf("Bearer %s", s.token), + } + + r := s.DoRawWithHeaders("POST", "/api/latest/fleet/software/package", b.Bytes(), expectedStatus, headers) + if expectedError != "" { + errMsg := extractServerErrorText(r.Body) + require.Contains(t, errMsg, expectedError) + } +} diff --git a/server/service/software_installers.go b/server/service/software_installers.go index 98ce9cd583..1e9b280ef3 100644 --- a/server/service/software_installers.go +++ b/server/service/software_installers.go @@ -140,3 +140,39 @@ func (svc *Service) DeleteSoftwareInstaller(ctx context.Context, id uint) error return fleet.ErrMissingLicense } + +///////////////////////////////////////////////////////////////////////////////// +// Request to install software in a host +///////////////////////////////////////////////////////////////////////////////// + +type installSoftwareRequest struct { + HostID uint `url:"host_id"` + SoftwareTitleID uint `url:"software_title_id"` +} + +type installSoftwareResponse struct { + Err error `json:"error,omitempty"` +} + +func (r installSoftwareResponse) error() error { return r.Err } + +func (r installSoftwareResponse) Status() int { return http.StatusAccepted } + +func installSoftwareTitleEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { + req := request.(*installSoftwareRequest) + + err := svc.InstallSoftwareTitle(ctx, req.HostID, req.SoftwareTitleID) + if err != nil { + return installSoftwareResponse{Err: err}, nil + } + + return installSoftwareResponse{}, nil +} + +func (svc *Service) InstallSoftwareTitle(ctx context.Context, hostID uint, softwareTitleID uint) error { + // skipauth: No authorization check needed due to implementation returning + // only license error. + svc.authz.SkipAuthorization(ctx) + + return fleet.ErrMissingLicense +} From 5d2d94401ea957d5dc5c37e4720e6b2a54a299a9 Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Thu, 2 May 2024 19:08:20 -0500 Subject: [PATCH 13/56] Add backend to download software installers (#18693) --- ee/server/service/software_installers.go | 84 +++++++++++++++++++++- server/fleet/service.go | 3 + server/fleet/software_installer.go | 9 ++- server/service/handler.go | 3 + server/service/integration_mdm_test.go | 57 ++++++++++++++- server/service/orbit.go | 45 +++++++++++- server/service/software_installers.go | 70 ++++++++++++++++++ server/service/software_installers_test.go | 79 ++++++++++++++++++++ 8 files changed, 346 insertions(+), 4 deletions(-) create mode 100644 server/service/software_installers_test.go diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index dd038ef9a6..39bc31108f 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -10,6 +10,7 @@ import ( "github.com/fleetdm/fleet/v4/pkg/file" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + hostctx "github.com/fleetdm/fleet/v4/server/contexts/host" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/go-kit/kit/log/level" ) @@ -58,7 +59,7 @@ func (svc *Service) UploadSoftwareInstaller(ctx context.Context, payload *fleet. if payload.InstallScript == "" { installerType := file.InstallerType(strings.TrimPrefix(filepath.Ext(payload.Filename), ".")) - installerPath := "some path" // TODO: where does this come from? + installerPath := payload.Filename // TODO: Confirm pending product input payload.InstallScript = file.GetInstallScript(installerType, installerPath) } @@ -111,6 +112,87 @@ func (svc *Service) DeleteSoftwareInstaller(ctx context.Context, id uint) error return nil } +func (svc *Service) GetSoftwareInstallerMetadata(ctx context.Context, installerID uint) (*fleet.SoftwareInstaller, error) { + // first do a basic authorization check, any logged in user can read teams + if err := svc.authz.Authorize(ctx, &fleet.Team{}, fleet.ActionRead); err != nil { + return nil, err + } + + // get the installer's metadata + meta, err := svc.ds.GetSoftwareInstallerMetadata(ctx, installerID) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "getting software installer metadata") + } + + // authorize with the software installer's team id + if err := svc.authz.Authorize(ctx, &fleet.SoftwareInstaller{TeamID: meta.TeamID}, fleet.ActionRead); err != nil { + return nil, err + } + + return meta, nil +} + +func (svc *Service) DownloadSoftwareInstaller(ctx context.Context, installerID uint) (*fleet.DownloadSoftwareInstallerPayload, error) { + meta, err := svc.GetSoftwareInstallerMetadata(ctx, installerID) + if err != nil { + return nil, err + } + + return svc.getSoftwareInstallerBinary(ctx, meta.StorageID, meta.Name) +} + +func (svc *Service) OrbitDownloadSoftwareInstaller(ctx context.Context, installerID uint) (*fleet.DownloadSoftwareInstallerPayload, error) { + // this is not a user-authenticated endpoint + svc.authz.SkipAuthorization(ctx) + + // TODO: confirm error handling + + host, ok := hostctx.FromContext(ctx) + if !ok { + return nil, fleet.OrbitError{Message: "internal error: missing host from request context"} + } + + // get the installer's metadata + meta, err := svc.ds.GetSoftwareInstallerMetadata(ctx, installerID) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "getting software installer metadata") + } + + // ensure it cannot get access to a different team's installer + var hTeamID uint + if host.TeamID != nil { + hTeamID = *host.TeamID + } + if (meta.TeamID != nil && *meta.TeamID != hTeamID) || (meta.TeamID == nil && hTeamID != 0) { + return nil, ctxerr.Wrap(ctx, fleet.OrbitError{}, "host team does not match installer team") + } + + return svc.getSoftwareInstallerBinary(ctx, meta.StorageID, meta.Name) +} + +func (svc *Service) getSoftwareInstallerBinary(ctx context.Context, storageID string, filename string) (*fleet.DownloadSoftwareInstallerPayload, error) { + // check if the installer exists in the store + exists, err := svc.softwareInstallStore.Exists(ctx, storageID) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "checking if installer exists") + } + if !exists { + return nil, ctxerr.Wrap(ctx, err, "does not exist in software installer store") + } + + // get the installer from the store + installer, size, err := svc.softwareInstallStore.Get(ctx, storageID) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "getting installer from store") + } + + return &fleet.DownloadSoftwareInstallerPayload{ + Filename: filename, + Installer: installer, + Size: size, + }, nil +} + func (svc *Service) InstallSoftwareTitle(ctx context.Context, hostID uint, softwareTitleID uint) error { // we need to use ds.Host because ds.HostLite doesn't return the orbit // node key diff --git a/server/fleet/service.go b/server/fleet/service.go index 3cbf73dd4f..b90bab4941 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -1006,4 +1006,7 @@ type Service interface { UploadSoftwareInstaller(ctx context.Context, payload *UploadSoftwareInstallerPayload) error DeleteSoftwareInstaller(ctx context.Context, id uint) error + GetSoftwareInstallerMetadata(ctx context.Context, id uint) (*SoftwareInstaller, error) + DownloadSoftwareInstaller(ctx context.Context, installerID uint) (*DownloadSoftwareInstallerPayload, error) + OrbitDownloadSoftwareInstaller(ctx context.Context, installerID uint) (*DownloadSoftwareInstallerPayload, error) } diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index 844617b976..cb72250781 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -50,7 +50,7 @@ type SoftwareInstaller struct { // UploadedAt is the time the software package was uploaded. UploadedAt time.Time `json:"uploaded_at" db:"uploaded_at"` // InstallerID is the unique identifier for the software package metadata in Fleet. - InstallerID uint `json:"-" db:"id"` + InstallerID uint `json:"installer_id" db:"id"` // InstallScript is the script to run to install the software package. InstallScript string `json:"install_script" db:"-"` // InstallScriptContentID is the ID of the install script content. @@ -137,6 +137,13 @@ type UploadSoftwareInstallerPayload struct { Source string } +// DownloadSoftwareInstallerPayload is the payload for downloading a software installer. +type DownloadSoftwareInstallerPayload struct { + Filename string + Installer io.ReadCloser + Size int64 +} + func SofwareInstallerSourceFromFilename(filename string) (string, error) { switch ext := filepath.Ext(filename); ext { case ".deb": diff --git a/server/service/handler.go b/server/service/handler.go index 76c9d55e9d..52e0f84d4d 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -373,6 +373,7 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC ue.POST("/api/v1/fleet/hosts/{host_id:[0-9]+}/software/install/{software_title_id:[0-9]+}", installSoftwareTitleEndpoint, installSoftwareRequest{}) // Sofware installers + ue.GET("/api/_version_/fleet/software/package/{id:[0-9]+}", getSoftwareInstallerEndpoint, getSoftwareInstallerRequest{}) ue.POST("/api/_version_/fleet/software/package", uploadSoftwareInstallerEndpoint, uploadSoftwareInstallerRequest{}) ue.DELETE("/api/_version_/fleet/software/package/{id:[0-9]+}", deleteSoftwareInstallerEndpoint, deleteSoftwareInstallerRequest{}) @@ -808,6 +809,8 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC oe.POST("/api/fleet/orbit/scripts/result", postOrbitScriptResultEndpoint, orbitPostScriptResultRequest{}) oe.PUT("/api/fleet/orbit/device_mapping", putOrbitDeviceMappingEndpoint, orbitPutDeviceMappingRequest{}) + oe.POST("/api/fleet/orbit/software_install/package", orbitDownloadSoftwareInstallerEndpoint, orbitDownloadSoftwareInstallerRequest{}) + oeWindowsMDM := oe.WithCustomMiddleware(mdmConfiguredMiddleware.VerifyWindowsMDM()) oeWindowsMDM.POST("/api/fleet/orbit/disk_encryption_key", postOrbitDiskEncryptionKeyEndpoint, orbitPostDiskEncryptionKeyRequest{}) diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 230c5509b8..86cbd2e3d8 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -8466,9 +8466,39 @@ func (s *integrationMDMTestSuite) TestIsServerBitlockerStatus() { require.Equal(t, fleet.DiskEncryptionEnforcing, *hr.Host.MDM.OSSettings.DiskEncryption.Status) } -func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadAndDelete() { +func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadDownloadAndDelete() { t := s.T() + openFile := func(name string) *os.File { + f, err := os.Open(filepath.Join("testdata", "software-installers", name)) + require.NoError(t, err) + return f + } + + var expectBytes []byte + var expectLen int + f := openFile("ruby.deb") + st, err := f.Stat() + require.NoError(t, err) + expectLen = int(st.Size()) + require.Equal(t, expectLen, 11340) + expectBytes = make([]byte, expectLen) + n, err := f.Read(expectBytes) + require.NoError(t, err) + require.Equal(t, n, expectLen) + f.Close() + + checkDownloadResponse := func(t *testing.T, r *http.Response, expectedFilename string) { + require.Equal(t, "application/octet-stream", r.Header.Get("Content-Type")) + require.Equal(t, fmt.Sprintf(`attachment;filename="%s"`, expectedFilename), r.Header.Get("Content-Disposition")) + require.NotZero(t, r.ContentLength) + require.Equal(t, expectLen, int(r.ContentLength)) + b, err := io.ReadAll(r.Body) + require.NoError(t, err) + require.Equal(t, expectLen, len(b)) + require.Equal(t, expectBytes, b) + } + checkSoftwareTitle := func(t *testing.T, title string, source string) uint { var id uint mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { @@ -8545,6 +8575,18 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadAndDelete() { // upload again fails s.uploadSoftwareInstaller(payload, http.StatusConflict, "already exists") + // download the installer + r := s.Do("GET", fmt.Sprintf("/api/latest/fleet/software/package/%d?alt=media", installerID), nil, http.StatusOK) + checkDownloadResponse(t, r, payload.Filename) + + // create an orbit host and request to download the installer + host := createOrbitEnrolledHost(t, "windows", "orbit-host", s.ds) + r = s.Do("POST", "/api/fleet/orbit/software_install/package?alt=media", orbitDownloadSoftwareInstallerRequest{ + InstallerID: installerID, + OrbitNodeKey: *host.OrbitNodeKey, + }, http.StatusOK) + checkDownloadResponse(t, r, payload.Filename) + // delete the installer s.Do("DELETE", fmt.Sprintf("/api/latest/fleet/software/package/%d", installerID), nil, http.StatusNoContent) }) @@ -8577,6 +8619,19 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadAndDelete() { // upload again fails s.uploadSoftwareInstaller(payload, http.StatusConflict, "already exists") + // download the installer + r := s.Do("GET", fmt.Sprintf("/api/latest/fleet/software/package/%d?alt=media", installerID), nil, http.StatusOK) + checkDownloadResponse(t, r, payload.Filename) + + // create an orbit host, assign to team and request to download the installer + host := createOrbitEnrolledHost(t, "windows", "orbit-host-team", s.ds) + require.NoError(t, s.ds.AddHostsToTeam(context.Background(), &createTeamResp.Team.ID, []uint{host.ID})) + r = s.Do("POST", "/api/fleet/orbit/software_install/package?alt=media", orbitDownloadSoftwareInstallerRequest{ + InstallerID: installerID, + OrbitNodeKey: *host.OrbitNodeKey, + }, http.StatusOK) + checkDownloadResponse(t, r, payload.Filename) + // delete the installer s.Do("DELETE", fmt.Sprintf("/api/latest/fleet/software/package/%d", installerID), nil, http.StatusNoContent) }) diff --git a/server/service/orbit.go b/server/service/orbit.go index 05230cabeb..e13bfd3b1b 100644 --- a/server/service/orbit.go +++ b/server/service/orbit.go @@ -344,7 +344,6 @@ func (svc *Service) GetOrbitConfig(ctx context.Context) (fleet.OrbitConfig, erro nudgeConfig, err = fleet.NewNudgeConfig(appConfig.MDM.MacOSUpdates) if err != nil { return fleet.OrbitConfig{}, err - } } } @@ -747,3 +746,47 @@ func (svc *Service) SetOrUpdateDiskEncryptionKey(ctx context.Context, encryption return nil } + +///////////////////////////////////////////////////////////////////////////////// +// Download Orbit software installer request +///////////////////////////////////////////////////////////////////////////////// + +type orbitDownloadSoftwareInstallerRequest struct { + Alt string `query:"alt"` + OrbitNodeKey string `json:"orbit_node_key"` + InstallerID uint `json:"installer_id"` +} + +// interface implementation required by the OrbitClient +func (r *orbitDownloadSoftwareInstallerRequest) setOrbitNodeKey(nodeKey string) { + r.OrbitNodeKey = nodeKey +} + +// interface implementation required by orbit authentication +func (r *orbitDownloadSoftwareInstallerRequest) orbitHostNodeKey() string { + return r.OrbitNodeKey +} + +func orbitDownloadSoftwareInstallerEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { + req := request.(*orbitDownloadSoftwareInstallerRequest) + + downloadRequested := req.Alt == "media" + if !downloadRequested { + // TODO: confirm error handling + return downloadSoftwareInstallerResponse{Err: &fleet.BadRequestError{Message: "only alt=media is supported"}}, nil + } + + p, err := svc.OrbitDownloadSoftwareInstaller(ctx, req.InstallerID) + if err != nil { + return downloadSoftwareInstallerResponse{Err: err}, nil + } + return downloadSoftwareInstallerResponse{payload: p}, nil +} + +func (svc *Service) OrbitDownloadSoftwareInstaller(ctx context.Context, installerID uint) (*fleet.DownloadSoftwareInstallerPayload, error) { + // skipauth: No authorization check needed due to implementation returning + // only license error. + svc.authz.SkipAuthorization(ctx) + + return nil, fleet.ErrMissingLicense +} diff --git a/server/service/software_installers.go b/server/service/software_installers.go index 1e9b280ef3..8ff62dbf75 100644 --- a/server/service/software_installers.go +++ b/server/service/software_installers.go @@ -3,11 +3,13 @@ package service import ( "context" "fmt" + "io" "mime/multipart" "net/http" "strconv" "github.com/docker/go-units" + "github.com/fleetdm/fleet/v4/server/contexts/logging" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/ptr" ) @@ -141,6 +143,74 @@ func (svc *Service) DeleteSoftwareInstaller(ctx context.Context, id uint) error return fleet.ErrMissingLicense } +type getSoftwareInstallerRequest struct { + Alt string `query:"alt,optional"` + InstallerID uint `url:"id"` +} + +type getSoftwareInstallerResponse struct { + // meta *fleet.SoftwareInstaller // NOTE: API design currently only supports downloading the + Err error `json:"error,omitempty"` +} + +func (r getSoftwareInstallerResponse) error() error { return r.Err } + +func getSoftwareInstallerEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { + req := request.(*getSoftwareInstallerRequest) + + downloadRequested := req.Alt == "media" + if !downloadRequested { + // TODO: confirm error handling + return getSoftwareInstallerResponse{Err: &fleet.BadRequestError{Message: "only alt=media is supported"}}, nil + } + + payload, err := svc.DownloadSoftwareInstaller(ctx, req.InstallerID) + if err != nil { + return downloadSoftwareInstallerResponse{Err: err}, nil + } + + return downloadSoftwareInstallerResponse{payload: payload}, nil +} + +func (svc *Service) GetSoftwareInstallerMetadata(ctx context.Context, id uint) (*fleet.SoftwareInstaller, error) { + // skipauth: No authorization check needed due to implementation returning + // only license error. + svc.authz.SkipAuthorization(ctx) + + return nil, fleet.ErrMissingLicense +} + +type downloadSoftwareInstallerResponse struct { + Err error `json:"error,omitempty"` + // fields used by hijackRender for the response. + payload *fleet.DownloadSoftwareInstallerPayload +} + +func (r downloadSoftwareInstallerResponse) error() error { return r.Err } + +func (r downloadSoftwareInstallerResponse) hijackRender(ctx context.Context, w http.ResponseWriter) { + w.Header().Set("Content-Length", strconv.Itoa(int(r.payload.Size))) + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment;filename="%s"`, r.payload.Filename)) + + // OK to just log the error here as writing anything on + // `http.ResponseWriter` sets the status code to 200 (and it can't be + // changed.) Clients should rely on matching content-length with the + // header provided + if n, err := io.Copy(w, r.payload.Installer); err != nil { + logging.WithExtras(ctx, "err", err, "bytes_copied", n) + } + r.payload.Installer.Close() +} + +func (svc *Service) DownloadSoftwareInstaller(ctx context.Context, id uint) (*fleet.DownloadSoftwareInstallerPayload, error) { + // skipauth: No authorization check needed due to implementation returning + // only license error. + svc.authz.SkipAuthorization(ctx) + + return nil, fleet.ErrMissingLicense +} + ///////////////////////////////////////////////////////////////////////////////// // Request to install software in a host ///////////////////////////////////////////////////////////////////////////////// diff --git a/server/service/software_installers_test.go b/server/service/software_installers_test.go new file mode 100644 index 0000000000..6e644c7a1f --- /dev/null +++ b/server/service/software_installers_test.go @@ -0,0 +1,79 @@ +package service + +import ( + "context" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/contexts/viewer" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/mock" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/fleetdm/fleet/v4/server/test" +) + +func TestSoftwareInstallersAuth(t *testing.T) { + ds := new(mock.Store) + + license := &fleet.LicenseInfo{Tier: fleet.TierPremium, Expiration: time.Now().Add(24 * time.Hour)} + + svc, ctx := newTestService(t, ds, nil, nil, &TestServerOpts{License: license}) + + testCases := []struct { + name string + user *fleet.User + teamID *uint + shouldFailRead bool + shouldFailWrite bool + }{ + {"no role no team", test.UserNoRoles, nil, true, true}, + {"no role team", test.UserNoRoles, ptr.Uint(1), true, true}, + {"global admin no team", test.UserAdmin, nil, false, false}, + {"global admin team", test.UserAdmin, ptr.Uint(1), false, false}, + {"global maintainer no team", test.UserMaintainer, nil, false, false}, + {"global maintainer team", test.UserMaintainer, ptr.Uint(1), false, false}, + {"global observer no team", test.UserObserver, nil, false, true}, + {"global observer team", test.UserObserver, ptr.Uint(1), false, true}, + {"global observer+ no team", test.UserObserverPlus, nil, false, true}, + {"global observer+ team", test.UserObserverPlus, ptr.Uint(1), false, true}, + {"global gitops no team", test.UserGitOps, nil, true, false}, + {"global gitops team", test.UserGitOps, ptr.Uint(1), true, false}, + {"team admin no team", test.UserTeamAdminTeam1, nil, true, true}, + {"team admin team", test.UserTeamAdminTeam1, ptr.Uint(1), false, false}, + {"team admin other team", test.UserTeamAdminTeam2, ptr.Uint(1), true, true}, + {"team maintainer no team", test.UserTeamMaintainerTeam1, nil, true, true}, + {"team maintainer team", test.UserTeamMaintainerTeam1, ptr.Uint(1), false, false}, + {"team maintainer other team", test.UserTeamMaintainerTeam2, ptr.Uint(1), true, true}, + {"team observer no team", test.UserTeamObserverTeam1, nil, true, true}, + {"team observer team", test.UserTeamObserverTeam1, ptr.Uint(1), false, true}, + {"team observer other team", test.UserTeamObserverTeam2, ptr.Uint(1), true, true}, + {"team observer+ no team", test.UserTeamObserverPlusTeam1, nil, true, true}, + {"team observer+ team", test.UserTeamObserverPlusTeam1, ptr.Uint(1), false, true}, + {"team observer+ other team", test.UserTeamObserverPlusTeam2, ptr.Uint(1), true, true}, + {"team gitops no team", test.UserTeamGitOpsTeam1, nil, true, true}, + {"team gitops team", test.UserTeamGitOpsTeam1, ptr.Uint(1), true, false}, + {"team gitops other team", test.UserTeamGitOpsTeam2, ptr.Uint(1), true, true}, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + ctx := viewer.NewContext(ctx, viewer.Viewer{User: tt.user}) + + ds.GetSoftwareInstallerMetadataFunc = func(ctx context.Context, installerID uint) (*fleet.SoftwareInstaller, error) { + return &fleet.SoftwareInstaller{TeamID: tt.teamID}, nil + } + + ds.DeleteSoftwareInstallerFunc = func(ctx context.Context, installerID uint) error { + return nil + } + + _, err := svc.DownloadSoftwareInstaller(ctx, 1) + checkAuthErr(t, tt.shouldFailRead, err) + + err = svc.DeleteSoftwareInstaller(ctx, 1) + checkAuthErr(t, tt.shouldFailWrite, err) + + // TODO: configure test with mock software installer store and add tests to check upload auth + }) + } +} From 2bae250ff72db4a3cd5f74f07684128128aa17ad Mon Sep 17 00:00:00 2001 From: Gabriel Hernandez Date: Fri, 3 May 2024 14:22:20 +0100 Subject: [PATCH 14/56] Feat UI upload software (#18575) relates to #18326 Add ability to add software from the UI. This includes - new button on software page to open add software modal - new add software modal to add software. > Note: still need to do form error validation but will do on another PR - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [x] Manual QA for all new/changed functionality --- changes/issue-18326-ui-add-software | 1 + frontend/components/Editor/Editor.tsx | 102 ++++++++ frontend/components/Editor/_styles.scss | 21 ++ frontend/components/Editor/index.ts | 1 + .../components/FileUploader/FileUploader.tsx | 82 +++++-- frontend/components/FileUploader/_styles.scss | 7 + frontend/components/FleetAce/FleetAce.tsx | 13 +- .../components/AddProfileModal.tsx | 3 + frontend/pages/SoftwarePage/SoftwarePage.tsx | 56 ++++- frontend/pages/SoftwarePage/_styles.scss | 6 + .../AddSoftwareAdvancedOptions.tsx | 110 +++++++++ .../AddSoftwareAdvancedOptions/_styles.scss | 17 ++ .../AddSoftwareAdvancedOptions/index.ts | 1 + .../AddSoftwareForm/AddSoftwareForm.tsx | 220 ++++++++++++++++++ .../components/AddSoftwareForm/_styles.scss | 43 ++++ .../components/AddSoftwareForm/helpers.ts | 168 +++++++++++++ .../components/AddSoftwareForm/index.ts | 1 + .../AddSoftwareModal/AddSoftwareModal.tsx | 109 +++++++++ .../components/AddSoftwareModal/index.ts | 1 + frontend/services/entities/software.ts | 20 ++ frontend/utilities/endpoints.ts | 1 + frontend/utilities/file/fileUtils.tests.ts | 16 ++ frontend/utilities/file/fileUtils.ts | 27 +++ 23 files changed, 986 insertions(+), 40 deletions(-) create mode 100644 changes/issue-18326-ui-add-software create mode 100644 frontend/components/Editor/Editor.tsx create mode 100644 frontend/components/Editor/_styles.scss create mode 100644 frontend/components/Editor/index.ts create mode 100644 frontend/pages/SoftwarePage/components/AddSoftwareAdvancedOptions/AddSoftwareAdvancedOptions.tsx create mode 100644 frontend/pages/SoftwarePage/components/AddSoftwareAdvancedOptions/_styles.scss create mode 100644 frontend/pages/SoftwarePage/components/AddSoftwareAdvancedOptions/index.ts create mode 100644 frontend/pages/SoftwarePage/components/AddSoftwareForm/AddSoftwareForm.tsx create mode 100644 frontend/pages/SoftwarePage/components/AddSoftwareForm/_styles.scss create mode 100644 frontend/pages/SoftwarePage/components/AddSoftwareForm/helpers.ts create mode 100644 frontend/pages/SoftwarePage/components/AddSoftwareForm/index.ts create mode 100644 frontend/pages/SoftwarePage/components/AddSoftwareModal/AddSoftwareModal.tsx create mode 100644 frontend/pages/SoftwarePage/components/AddSoftwareModal/index.ts create mode 100644 frontend/utilities/file/fileUtils.tests.ts create mode 100644 frontend/utilities/file/fileUtils.ts diff --git a/changes/issue-18326-ui-add-software b/changes/issue-18326-ui-add-software new file mode 100644 index 0000000000..297caae502 --- /dev/null +++ b/changes/issue-18326-ui-add-software @@ -0,0 +1 @@ +- add ability to upload software from the UI diff --git a/frontend/components/Editor/Editor.tsx b/frontend/components/Editor/Editor.tsx new file mode 100644 index 0000000000..4ab4fab32f --- /dev/null +++ b/frontend/components/Editor/Editor.tsx @@ -0,0 +1,102 @@ +import classnames from "classnames"; +import React, { ReactNode } from "react"; +import AceEditor from "react-ace"; + +const baseClass = "editor"; + +interface IEditorProps { + focus?: boolean; + label?: string; + error?: string | null; + /** + * Help text to display below the editor. + */ + helpText?: ReactNode; + /** Sets the value of the input. Use this if you'd like the editor + * to be a controlled component */ + value?: string; + /** Sets the default value of the input. Use this if you'd like the editor + * to be an uncontrolled component */ + defaultValue?: string; + /** Enabled wrapping lines. + * @default false + */ + wrapEnabled?: boolean; + /** A unique name for the editor. + * @default "editor" + */ + name?: string; + maxLines?: number; + className?: string; + onChange: (value: string, event?: any) => void; +} + +/** + * This component is a generic editor that uses the AceEditor component. + * TODO: We should move FleetAce and YamlAce into here and deprecate importing + * them directly. This component should be used for all editor components and + * be configurable from the props. We should look into dynmaic imports for + * this. + */ +const Editor = ({ + helpText, + label, + error, + focus, + value, + defaultValue, + wrapEnabled = false, + name = "editor", + maxLines = 20, + className, + onChange, +}: IEditorProps) => { + const classNames = classnames(baseClass, className, { + [`${baseClass}__error`]: !!error, + }); + + const renderLabel = () => { + const labelText = error || label; + const labelClassName = classnames(`${baseClass}__label`, { + [`${baseClass}__label--error`]: !!error, + }); + + if (!labelText) { + return null; + } + + return

{labelText}
; + }; + + const renderHelpText = () => { + if (helpText) { + return
{helpText}
; + } + return null; + }; + + return ( +
+ {renderLabel()} + + {renderHelpText()} +
+ ); +}; + +export default Editor; diff --git a/frontend/components/Editor/_styles.scss b/frontend/components/Editor/_styles.scss new file mode 100644 index 0000000000..22fbacfd33 --- /dev/null +++ b/frontend/components/Editor/_styles.scss @@ -0,0 +1,21 @@ +.editor { + + &__label { + font-size: $x-small; + font-weight: $bold; + + &--error { + color: $core-vibrant-red; + } + } + + &__help-text { + @include help-text; + } + + &__error { + .ace-fleet { + border: 1px solid $core-vibrant-red; + } + } +} diff --git a/frontend/components/Editor/index.ts b/frontend/components/Editor/index.ts new file mode 100644 index 0000000000..100d029231 --- /dev/null +++ b/frontend/components/Editor/index.ts @@ -0,0 +1 @@ +export { default } from "./Editor"; diff --git a/frontend/components/FileUploader/FileUploader.tsx b/frontend/components/FileUploader/FileUploader.tsx index 52ab307656..759f9c9501 100644 --- a/frontend/components/FileUploader/FileUploader.tsx +++ b/frontend/components/FileUploader/FileUploader.tsx @@ -1,10 +1,11 @@ -import React from "react"; +import React, { ReactNode, useState } from "react"; import classnames from "classnames"; import Button from "components/buttons/Button"; import Card from "components/Card"; import { GraphicNames } from "components/graphics"; import Graphic from "components/Graphic"; +import Icon from "components/Icon"; const baseClass = "file-uploader"; @@ -32,9 +33,20 @@ interface IFileUploaderProps { * https://developer.mozilla.org/en-US/docs/Web/HTML/Attributes/accept */ accept?: string; - /** The text to display on the upload button */ + /** The text to display on the upload button + * @default "Upload" + */ buttonMessage?: string; className?: string; + /** renders the button to open the file uploader to appear as a button or + * a link. + * @default "button" + */ + buttonType?: "button" | "link"; + /** If provided FileUploader will display this component when the file is + * selected. This is used for previewing the file before uploading. + */ + filePreview?: ReactNode; onFileUpload: (files: FileList | null) => void; } @@ -47,11 +59,26 @@ const FileUploader = ({ additionalInfo, isLoading = false, accept, - buttonMessage = "Upload", + filePreview, className, + buttonMessage = "Upload", + buttonType = "button", onFileUpload, }: IFileUploaderProps) => { - const classes = classnames(baseClass, className); + const [isFileSelected, setIsFileSelected] = useState(false); + + const classes = classnames(baseClass, className, { + [`${baseClass}__file-preview`]: filePreview !== undefined && isFileSelected, + }); + const buttonVariant = buttonType === "button" ? "brand" : "text-icon"; + + const onFileSelect = (e: React.ChangeEvent) => { + const files = e.target.files; + onFileUpload(files); + setIsFileSelected(true); + + e.target.value = ""; + }; const renderGraphics = () => { const graphicNamesArr = @@ -64,29 +91,36 @@ const FileUploader = ({ /> )); }; + return ( -
{renderGraphics()}
-

{message}

- {additionalInfo && ( -

{additionalInfo}

+ {isFileSelected && filePreview ? ( + filePreview + ) : ( + <> +
{renderGraphics()}
+

{message}

+ {additionalInfo && ( +

{additionalInfo}

+ )} + + + )} - - { - onFileUpload(e.target.files); - e.target.value = ""; - }} - />
); }; diff --git a/frontend/components/FileUploader/_styles.scss b/frontend/components/FileUploader/_styles.scss index 4a6835d3ec..4cb2e24539 100644 --- a/frontend/components/FileUploader/_styles.scss +++ b/frontend/components/FileUploader/_styles.scss @@ -10,6 +10,12 @@ text-align: center; gap: $pad-small; + // when the file preview is showing, we want the padding to be + // slightly smaller on the top and bottom. + &__file-preview { + padding: $pad-medium $pad-large; + } + &__graphics { display: flex; align-items: center; @@ -39,6 +45,7 @@ display: flex; align-items: center; justify-content: center; + gap: $pad-small; &:hover { cursor: pointer; diff --git a/frontend/components/FleetAce/FleetAce.tsx b/frontend/components/FleetAce/FleetAce.tsx index 5a6f6ba7bc..c30232cb59 100644 --- a/frontend/components/FleetAce/FleetAce.tsx +++ b/frontend/components/FleetAce/FleetAce.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useRef } from "react"; +import React, { ReactNode, useCallback, useRef } from "react"; import AceEditor from "react-ace"; import ReactAce from "react-ace/lib/ace"; import { IAceEditor } from "react-ace/lib/types"; @@ -30,10 +30,13 @@ export interface IFleetAceProps { name?: string; value?: string; readOnly?: boolean; + maxLines?: number; showGutter?: boolean; wrapEnabled?: boolean; + /** @deprecated use the prop `className` instead */ wrapperClassName?: string; - helpText?: string; + className?: string; + helpText?: ReactNode; labelActionComponent?: React.ReactNode; style?: React.CSSProperties; onBlur?: (editor?: IAceEditor) => void; @@ -53,9 +56,11 @@ const FleetAce = ({ name = "query-editor", value, readOnly, + maxLines = 20, showGutter = true, wrapEnabled = false, wrapperClassName, + className, helpText, style, onBlur, @@ -64,7 +69,7 @@ const FleetAce = ({ handleSubmit = noop, }: IFleetAceProps): JSX.Element => { const editorRef = useRef(null); - const wrapperClass = classnames(wrapperClassName, baseClass, { + const wrapperClass = classnames(className, wrapperClassName, baseClass, { [`${baseClass}__wrapper--error`]: !!error, }); @@ -250,7 +255,7 @@ const FleetAce = ({ fontSize={fontSize} mode="fleet" minLines={2} - maxLines={20} + maxLines={maxLines} name={name} onChange={onChange} onBlur={onBlurHandler} diff --git a/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileUploader/components/AddProfileModal.tsx b/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileUploader/components/AddProfileModal.tsx index 403ef83c16..b948e9d9dd 100644 --- a/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileUploader/components/AddProfileModal.tsx +++ b/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileUploader/components/AddProfileModal.tsx @@ -62,6 +62,9 @@ const FileChooser = ({ ); +// TODO: if we reuse this one more time, we should consider moving this +// into FileUploader as a default preview. Currently we have this in +// AddSoftwareForm.tsx and here. const FileDetails = ({ baseClass, details: { name, platform }, diff --git a/frontend/pages/SoftwarePage/SoftwarePage.tsx b/frontend/pages/SoftwarePage/SoftwarePage.tsx index 20ccb325e4..89020ca581 100644 --- a/frontend/pages/SoftwarePage/SoftwarePage.tsx +++ b/frontend/pages/SoftwarePage/SoftwarePage.tsx @@ -28,6 +28,7 @@ import TeamsHeader from "components/TeamsHeader"; import TabsWrapper from "components/TabsWrapper"; import ManageAutomationsModal from "./components/ManageSoftwareAutomationsModal"; +import AddSoftwareModal from "./components/AddSoftwareModal"; interface ISoftwareSubNavItem { name: string; @@ -110,6 +111,8 @@ const SoftwarePage = ({ children, router, location }: ISoftwarePageProps) => { isGlobalAdmin, isGlobalMaintainer, isOnGlobalTeam, + isTeamAdmin, + isTeamMaintainer, isPremiumTier, isSandboxMode, } = useContext(AppContext); @@ -142,6 +145,7 @@ const SoftwarePage = ({ children, router, location }: ISoftwarePageProps) => { ); const [showPreviewPayloadModal, setShowPreviewPayloadModal] = useState(false); const [showPreviewTicketModal, setShowPreviewTicketModal] = useState(false); + const [showAddSoftwareModal, setShowAddSoftwareModal] = useState(false); const { currentTeamId, @@ -218,13 +222,14 @@ const SoftwarePage = ({ children, router, location }: ISoftwarePageProps) => { const isSoftwareConfigLoaded = !isFetchingSoftwareConfig && !softwareConfigError && !!softwareConfig; - const canManageAutomations = - isGlobalAdmin && (!isPremiumTier || !isAnyTeamSelected); - const toggleManageAutomationsModal = useCallback(() => { setShowManageAutomationsModal(!showManageAutomationsModal); }, [setShowManageAutomationsModal, showManageAutomationsModal]); + const toggleAddSoftwareModal = useCallback(() => { + setShowAddSoftwareModal(!showAddSoftwareModal); + }, [showAddSoftwareModal]); + const togglePreviewPayloadModal = useCallback(() => { setShowPreviewPayloadModal(!showPreviewPayloadModal); }, [setShowPreviewPayloadModal, showPreviewPayloadModal]); @@ -295,6 +300,35 @@ const SoftwarePage = ({ children, router, location }: ISoftwarePageProps) => { ); }; + const renderPageActions = () => { + const canManageAutomations = + isGlobalAdmin && (!isPremiumTier || !isAnyTeamSelected); + + const canAddSoftware = + isGlobalAdmin || isGlobalMaintainer || isTeamAdmin || isTeamMaintainer; + + if (!isSoftwareConfigLoaded) return null; + + return ( +
+ {canManageAutomations && ( + + )} + {canAddSoftware && ( + + )} +
+ ); + }; + const renderHeaderDescription = () => { return (

@@ -358,15 +392,7 @@ const SoftwarePage = ({ children, router, location }: ISoftwarePageProps) => {

{renderTitle()}
- {canManageAutomations && isSoftwareConfigLoaded && ( - - )} + {renderPageActions()}
{renderHeaderDescription()} @@ -386,6 +412,12 @@ const SoftwarePage = ({ children, router, location }: ISoftwarePageProps) => { recentVulnerabilityMaxAge={recentVulnerabilityMaxAge} /> )} + {showAddSoftwareModal && ( + + )}
); diff --git a/frontend/pages/SoftwarePage/_styles.scss b/frontend/pages/SoftwarePage/_styles.scss index cfdd0b9c51..2740a08de4 100644 --- a/frontend/pages/SoftwarePage/_styles.scss +++ b/frontend/pages/SoftwarePage/_styles.scss @@ -25,6 +25,12 @@ } } + &__action-buttons { + display: flex; + align-items: center; + gap: $pad-medium; + } + &__text { margin-right: $pad-large; } diff --git a/frontend/pages/SoftwarePage/components/AddSoftwareAdvancedOptions/AddSoftwareAdvancedOptions.tsx b/frontend/pages/SoftwarePage/components/AddSoftwareAdvancedOptions/AddSoftwareAdvancedOptions.tsx new file mode 100644 index 0000000000..96de54fd33 --- /dev/null +++ b/frontend/pages/SoftwarePage/components/AddSoftwareAdvancedOptions/AddSoftwareAdvancedOptions.tsx @@ -0,0 +1,110 @@ +import React, { useState } from "react"; + +import Editor from "components/Editor"; +import CustomLink from "components/CustomLink"; +import FleetAce from "components/FleetAce"; +import RevealButton from "components/buttons/RevealButton"; +import Checkbox from "components/forms/fields/Checkbox"; + +const baseClass = "add-software-advanced-options"; + +interface IAddSoftwareAdvancedOptionsProps { + errors: { preInstallCondition?: string; postInstallScript?: string }; + showPreInstallCondition: boolean; + showPostInstallScript: boolean; + preInstallCondition?: string; + postInstallScript?: string; + onTogglePreInstallCondition: (value: boolean) => void; + onTogglePostInstallScript: (value: boolean) => void; + onChangePreInstallCondition: (value?: string) => void; + onChangePostInstallScript: (value?: string) => void; +} + +const AddSoftwareAdvancedOptions = ({ + errors, + showPreInstallCondition, + showPostInstallScript, + preInstallCondition, + postInstallScript, + onTogglePreInstallCondition, + onTogglePostInstallScript, + onChangePreInstallCondition, + onChangePostInstallScript, +}: IAddSoftwareAdvancedOptionsProps) => { + const [showAdvancedOptions, setShowAdvancedOptions] = useState(false); + + const onChangePreInstallCheckbox = () => { + onTogglePreInstallCondition(!showPreInstallCondition); + }; + + const onChangePostInstallCheckbox = () => { + onTogglePostInstallScript(!showPostInstallScript); + }; + + return ( +
+ setShowAdvancedOptions(!showAdvancedOptions)} + /> + {showAdvancedOptions && ( +
+ + Pre-install condition + + {showPreInstallCondition && ( + + Software will be installed only if the{" "} + + + } + /> + )} + + Post-install script + + {showPostInstallScript && ( + <> + + + )} +
+ )} +
+ ); +}; + +export default AddSoftwareAdvancedOptions; diff --git a/frontend/pages/SoftwarePage/components/AddSoftwareAdvancedOptions/_styles.scss b/frontend/pages/SoftwarePage/components/AddSoftwareAdvancedOptions/_styles.scss new file mode 100644 index 0000000000..58f1f85892 --- /dev/null +++ b/frontend/pages/SoftwarePage/components/AddSoftwareAdvancedOptions/_styles.scss @@ -0,0 +1,17 @@ +.add-software-advanced-options { + display: flex; + flex-direction: column; + align-items: flex-start; + gap: $pad-large; + + &__input-fields { + width: 100%; + display: flex; + flex-direction: column; + gap: $pad-medium; + } + + &__table-link { + font-size: $xx-small; + } +} diff --git a/frontend/pages/SoftwarePage/components/AddSoftwareAdvancedOptions/index.ts b/frontend/pages/SoftwarePage/components/AddSoftwareAdvancedOptions/index.ts new file mode 100644 index 0000000000..264fa61b11 --- /dev/null +++ b/frontend/pages/SoftwarePage/components/AddSoftwareAdvancedOptions/index.ts @@ -0,0 +1 @@ +export { default } from "./AddSoftwareAdvancedOptions"; diff --git a/frontend/pages/SoftwarePage/components/AddSoftwareForm/AddSoftwareForm.tsx b/frontend/pages/SoftwarePage/components/AddSoftwareForm/AddSoftwareForm.tsx new file mode 100644 index 0000000000..5c619e9687 --- /dev/null +++ b/frontend/pages/SoftwarePage/components/AddSoftwareForm/AddSoftwareForm.tsx @@ -0,0 +1,220 @@ +import React, { useState } from "react"; + +// @ts-ignore +import InputField from "components/forms/fields/InputField"; +import Spinner from "components/Spinner"; +import Button from "components/buttons/Button"; +import FileUploader from "components/FileUploader"; +import Graphic from "components/Graphic"; + +import AddSoftwareAdvancedOptions from "../AddSoftwareAdvancedOptions"; + +import { + generateFormValidation, + getFileDetails, + getInstallScript, +} from "./helpers"; + +const baseClass = "add-software-form"; + +const UploadingSoftware = () => { + return ( +
+ +

Uploading. It may take few minutes to finish.

+
+ ); +}; + +// TODO: if we reuse this one more time, we should consider moving this +// into FileUploader as a default preview. Currently we have this in +// AddProfileModal.tsx and here. +const FileDetails = ({ + details: { name, platform }, +}: { + details: { + name: string; + platform: string; + }; +}) => ( +
+ +
+
{name}
+
+ {platform} +
+
+
+); + +export interface IAddSoftwareFormData { + software: File | null; + installScript: string; + preInstallCondition?: string; + postInstallScript?: string; +} + +export interface IFormValidation { + isValid: boolean; + software: { isValid: boolean }; + preInstallCondition?: { isValid: boolean; message?: string }; + postInstallScript?: { isValid: boolean; message?: string }; +} + +interface IAddSoftwareFormProps { + isUploading: boolean; + onCancel: () => void; + onSubmit: (formData: IAddSoftwareFormData) => void; +} + +const AddSoftwareForm = ({ + isUploading, + onCancel, + onSubmit, +}: IAddSoftwareFormProps) => { + console.log("rerender"); + const [showPreInstallCondition, setShowPreInstallCondition] = useState(false); + const [showPostInstallScript, setShowPostInstallScript] = useState(false); + const [formData, setFormData] = useState({ + software: null, + installScript: "", + preInstallCondition: undefined, + postInstallScript: undefined, + }); + const [formValidation, setFormValidation] = useState({ + isValid: false, + software: { isValid: false }, + }); + + const onFileUpload = (files: FileList | null) => { + if (files && files.length > 0) { + const file = files[0]; + const newData = { + ...formData, + software: file, + installScript: getInstallScript(file), + }; + setFormData(newData); + setFormValidation( + generateFormValidation( + newData, + showPreInstallCondition, + showPostInstallScript + ) + ); + } + }; + + const onFormSubmit = (evt: React.FormEvent) => { + evt.preventDefault(); + onSubmit(formData); + }; + + const onTogglePreInstallConditionCheckbox = (value: boolean) => { + const newData = { ...formData, preInstallCondition: undefined }; + setShowPreInstallCondition(value); + setFormData(newData); + setFormValidation( + generateFormValidation(newData, value, showPostInstallScript) + ); + }; + + const onTogglePostInstallScriptCheckbox = (value: boolean) => { + const newData = { ...formData, postInstallScript: undefined }; + setShowPostInstallScript(value); + setFormData(newData); + setFormValidation( + generateFormValidation(newData, showPreInstallCondition, value) + ); + }; + + const onChangeInstallScript = (value: string) => { + setFormData({ ...formData, installScript: value }); + }; + + const onChangePreInstallCondition = (value?: string) => { + const newData = { ...formData, preInstallCondition: value }; + setFormData(newData); + setFormValidation( + generateFormValidation( + newData, + showPreInstallCondition, + showPostInstallScript + ) + ); + }; + + const onChangePostInstallScript = (value?: string) => { + const newData = { ...formData, postInstallScript: value }; + setFormData(newData); + setFormValidation( + generateFormValidation( + newData, + showPreInstallCondition, + showPostInstallScript + ) + ); + }; + + const isSubmitDisabled = !formValidation.isValid; + + return ( +
+ {isUploading ? ( + + ) : ( +
+ + ) + } + /> + {formData.software && ( + + )} + +
+ + +
+ + )} +
+ ); +}; + +export default AddSoftwareForm; diff --git a/frontend/pages/SoftwarePage/components/AddSoftwareForm/_styles.scss b/frontend/pages/SoftwarePage/components/AddSoftwareForm/_styles.scss new file mode 100644 index 0000000000..d955a1df9c --- /dev/null +++ b/frontend/pages/SoftwarePage/components/AddSoftwareForm/_styles.scss @@ -0,0 +1,43 @@ +.add-software-form { + + &__uploading-message { + display: flex; + align-items: center; + flex-direction: column; + gap: $pad-large; + + p { + margin: 0 + } + } + + &__form { + display: flex; + flex-direction: column; + gap: $pad-large; + } + + &__file-uploader { + box-sizing: border-box; + } + + &__selected-file { + display: flex; + gap: $pad-medium; + align-items: center; + width: 100%; + text-align: left; + + &--details { + &--name { + font-size: $x-small; + font-weight: $bold; + } + + &--platform { + font-size: $xx-small; + color: $ui-fleet-black-75; + } + } + } +} diff --git a/frontend/pages/SoftwarePage/components/AddSoftwareForm/helpers.ts b/frontend/pages/SoftwarePage/components/AddSoftwareForm/helpers.ts new file mode 100644 index 0000000000..3464a83153 --- /dev/null +++ b/frontend/pages/SoftwarePage/components/AddSoftwareForm/helpers.ts @@ -0,0 +1,168 @@ +import validator from "validator"; + +// @ts-ignore +import validateQuery from "components/forms/validators/validate_query"; +import { getPlatformDisplayName } from "utilities/file/fileUtils"; + +import { IAddSoftwareFormData, IFormValidation } from "./AddSoftwareForm"; + +type IAddSoftwareFormValidatorKey = Exclude< + keyof IAddSoftwareFormData, + "installScript" +>; + +type IMessageFunc = (formData: IAddSoftwareFormData) => string; +type IValidationMessage = string | IMessageFunc; + +interface IValidation { + name: string; + isValid: ( + formData: IAddSoftwareFormData, + enabledPreInstallCondition?: boolean, + enabledPostInstallScript?: boolean + ) => boolean; + message?: IValidationMessage; +} + +/** configuration defines validations for each filed in the form. It defines rules + * to determine if a field is valid, and rules for generating an error message. + */ +const FORM_VALIDATION_CONFIG: Record< + IAddSoftwareFormValidatorKey, + { validations: IValidation[] } +> = { + software: { + validations: [ + { + name: "required", + isValid: (formData) => formData.software !== null, + }, + ], + }, + preInstallCondition: { + validations: [ + { + name: "required", + isValid: ( + formData: IAddSoftwareFormData, + enabledPreInstallCondition + ) => { + if (!enabledPreInstallCondition) { + return true; + } + return ( + formData.preInstallCondition !== undefined && + !validator.isEmpty(formData.preInstallCondition) + ); + }, + message: (formData) => { + // we dont want an error message until the user has interacted with + // the field. This is why we check for undefined here. + if (formData.preInstallCondition === undefined) { + return ""; + } + return "Pre-install condition is required when enabled."; + }, + }, + { + name: "invalidQuery", + isValid: (formData, enabledPreInstallCondition) => { + if (!enabledPreInstallCondition) { + return true; + } + return ( + formData.preInstallCondition !== undefined && + validateQuery(formData.preInstallCondition).valid + ); + }, + message: (formData) => + validateQuery(formData.preInstallCondition).error, + }, + ], + }, + postInstallScript: { + validations: [ + { + name: "required", + message: (formData) => { + // we dont want an error message until the user has interacted with + // the field. This is why we check for undefined here. + if (formData.postInstallScript === undefined) { + return ""; + } + return "Post-install script is required when enabled."; + }, + isValid: (formData, _, enabledPostInstallScript) => { + if (!enabledPostInstallScript) { + return true; + } + return ( + formData.postInstallScript !== undefined && + !validator.isEmpty(formData.postInstallScript) + ); + }, + }, + ], + }, +}; + +const getErrorMessage = ( + formData: IAddSoftwareFormData, + message?: IValidationMessage +) => { + if (message === undefined || typeof message === "string") { + return message; + } + return message(formData); +}; + +export const generateFormValidation = ( + formData: IAddSoftwareFormData, + showingPreInstallCondition: boolean, + showingPostInstallScript: boolean +) => { + const formValidation: IFormValidation = { + isValid: true, + software: { + isValid: false, + }, + }; + + Object.keys(FORM_VALIDATION_CONFIG).forEach((key) => { + const objKey = key as keyof typeof FORM_VALIDATION_CONFIG; + const failedValidation = FORM_VALIDATION_CONFIG[objKey].validations.find( + (validation) => + !validation.isValid( + formData, + showingPreInstallCondition, + showingPostInstallScript + ) + ); + + if (!failedValidation) { + formValidation[objKey] = { + isValid: true, + }; + } else { + formValidation.isValid = false; + formValidation[objKey] = { + isValid: false, + message: getErrorMessage(formData, failedValidation.message), + }; + } + }); + + return formValidation; +}; + +export const getFileDetails = (file: File) => { + return { + name: file.name, + platform: getPlatformDisplayName(file), + }; +}; + +export const getInstallScript = (file: File) => { + // TODO: get this dynamically + return `sudo installer -pkg ${file.name} -target /`; +}; diff --git a/frontend/pages/SoftwarePage/components/AddSoftwareForm/index.ts b/frontend/pages/SoftwarePage/components/AddSoftwareForm/index.ts new file mode 100644 index 0000000000..d3ea76d47d --- /dev/null +++ b/frontend/pages/SoftwarePage/components/AddSoftwareForm/index.ts @@ -0,0 +1 @@ +export { default } from "./AddSoftwareForm"; diff --git a/frontend/pages/SoftwarePage/components/AddSoftwareModal/AddSoftwareModal.tsx b/frontend/pages/SoftwarePage/components/AddSoftwareModal/AddSoftwareModal.tsx new file mode 100644 index 0000000000..e2233964d6 --- /dev/null +++ b/frontend/pages/SoftwarePage/components/AddSoftwareModal/AddSoftwareModal.tsx @@ -0,0 +1,109 @@ +import React, { useContext, useEffect, useState } from "react"; + +import { APP_CONTEXT_ALL_TEAMS_ID } from "interfaces/team"; +import { getErrorReason } from "interfaces/errors"; +import softwareAPI from "services/entities/software"; +import { NotificationContext } from "context/notification"; + +import Modal from "components/Modal"; +import Button from "components/buttons/Button"; + +import AddSoftwareForm from "../AddSoftwareForm"; +import { IAddSoftwareFormData } from "../AddSoftwareForm/AddSoftwareForm"; + +// 2 minutes +const UPLOAD_TIMEOUT = 120000; + +const baseClass = "add-software-modal"; + +interface IAllTeamsMessageProps { + onExit: () => void; +} + +const AllTeamsMessage = ({ onExit }: IAllTeamsMessageProps) => { + return ( + <> +

+ Please select a team first. Software can't be added when{" "} + All teams is selected. +

+
+ +
+ + ); +}; + +interface IAddSoftwareModalProps { + teamId: number; + onExit: () => void; +} + +const AddSoftwareModal = ({ teamId, onExit }: IAddSoftwareModalProps) => { + const { renderFlash } = useContext(NotificationContext); + const [isUploading, setIsUploading] = useState(false); + + useEffect(() => { + let timeout: NodeJS.Timeout; + + const beforeUnloadHandler = (e: BeforeUnloadEvent) => { + e.preventDefault(); + // Included for legacy support, e.g. Chrome/Edge < 119 + e.returnValue = true; + }; + + // set up event listener to prevent user from leaving page while uploading + if (isUploading) { + addEventListener("beforeunload", beforeUnloadHandler); + timeout = setTimeout(() => { + removeEventListener("beforeunload", beforeUnloadHandler); + }, UPLOAD_TIMEOUT); + } else { + removeEventListener("beforeunload", beforeUnloadHandler); + } + + // clean up event listener and timeout on component unmount + return () => { + removeEventListener("beforeunload", beforeUnloadHandler); + clearTimeout(timeout); + }; + }, [isUploading]); + + const onAddSoftware = async (formData: IAddSoftwareFormData) => { + setIsUploading(true); + + try { + await softwareAPI.addSoftwarePackage(formData, teamId); + renderFlash("success", "Software added successfully!"); // TODO: change message + } catch (e) { + renderFlash("error", getErrorReason(e)); + } + + setIsUploading(false); + }; + + return ( + + <> + {teamId === APP_CONTEXT_ALL_TEAMS_ID ? ( + + ) : ( + + )} + + + ); +}; + +export default AddSoftwareModal; diff --git a/frontend/pages/SoftwarePage/components/AddSoftwareModal/index.ts b/frontend/pages/SoftwarePage/components/AddSoftwareModal/index.ts new file mode 100644 index 0000000000..d8ac7200d6 --- /dev/null +++ b/frontend/pages/SoftwarePage/components/AddSoftwareModal/index.ts @@ -0,0 +1 @@ +export { default } from "./AddSoftwareModal"; diff --git a/frontend/services/entities/software.ts b/frontend/services/entities/software.ts index 3643bab792..3cadcdfd3d 100644 --- a/frontend/services/entities/software.ts +++ b/frontend/services/entities/software.ts @@ -10,6 +10,7 @@ import { ISoftwareTitle, } from "interfaces/software"; import { buildQueryStringFromParams, QueryParams } from "utilities/url"; +import { IAddSoftwareFormData } from "pages/SoftwarePage/components/AddSoftwareForm/AddSoftwareForm"; export interface ISoftwareApiParams { page?: number; @@ -186,4 +187,23 @@ export default { return sendRequest("GET", path); }, + + addSoftwarePackage: (data: IAddSoftwareFormData, teamId?: number) => { + const { SOFTWARE_PACKAGE } = endpoints; + + if (!data.software) { + throw new Error("Software package is required"); + } + + const formData = new FormData(); + formData.append("software", data.software); + data.installScript && formData.append("install_script", data.installScript); + data.preInstallCondition && + formData.append("pre_install_query", data.preInstallCondition); + data.postInstallScript && + formData.append("post_install_script", data.postInstallScript); + teamId && formData.append("team_id", teamId.toString()); + + return sendRequest("POST", SOFTWARE_PACKAGE, formData); + }, }; diff --git a/frontend/utilities/endpoints.ts b/frontend/utilities/endpoints.ts index 5b82ebfbe2..07cc0b3245 100644 --- a/frontend/utilities/endpoints.ts +++ b/frontend/utilities/endpoints.ts @@ -128,6 +128,7 @@ export default { SOFTWARE_VERSIONS: `/${API_VERSION}/fleet/software/versions`, SOFTWARE_VERSION: (id: number) => `/${API_VERSION}/fleet/software/versions/${id}`, + SOFTWARE_PACKAGE: `/${API_VERSION}/fleet/software/package`, SSO: `/v1/fleet/sso`, STATUS_LABEL_COUNTS: `/${API_VERSION}/fleet/host_summary`, diff --git a/frontend/utilities/file/fileUtils.tests.ts b/frontend/utilities/file/fileUtils.tests.ts new file mode 100644 index 0000000000..8d65063892 --- /dev/null +++ b/frontend/utilities/file/fileUtils.tests.ts @@ -0,0 +1,16 @@ +import { getPlatformDisplayName } from "./fileUtils"; + +describe("fileUtils", () => { + describe("getPlatformDisplayName", () => { + it("should return the correct platform display name depending on the file extension", () => { + const file = new File([""], "test.pkg"); + expect(getPlatformDisplayName(file)).toEqual("macOS"); + + const file2 = new File([""], "test.exe"); + expect(getPlatformDisplayName(file2)).toEqual("Windows"); + + const file3 = new File([""], "test.deb"); + expect(getPlatformDisplayName(file3)).toEqual("linux"); + }); + }); +}); diff --git a/frontend/utilities/file/fileUtils.ts b/frontend/utilities/file/fileUtils.ts new file mode 100644 index 0000000000..9cecb327a1 --- /dev/null +++ b/frontend/utilities/file/fileUtils.ts @@ -0,0 +1,27 @@ +type IPlatformDisplayName = "macOS" | "Windows" | "linux"; + +const getFileExtension = (file: File) => { + const nameParts = file.name.split("."); + return nameParts.slice(-1)[0]; +}; + +export const FILE_EXTENSIONS_TO_PLATFORM_DISPLAY_NAME: Record< + string, + IPlatformDisplayName +> = { + json: "macOS", + pkg: "macOS", + mobileconfig: "macOS", + exe: "Windows", + msi: "Windows", + xml: "Windows", + deb: "linux", +}; + +/** + * This gets the platform display name from the file. + */ +export const getPlatformDisplayName = (file: File) => { + const fileExt = getFileExtension(file); + return FILE_EXTENSIONS_TO_PLATFORM_DISPLAY_NAME[fileExt]; +}; From 874c3cd8119278e5301f6faf8b12e418923ffe4f Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Fri, 3 May 2024 12:03:59 -0400 Subject: [PATCH 15/56] Add orbit endpoint to receive results of a software installation attempt (#18689) #18675 --- ...rbit-endpoint-for-software-install-results | 1 + server/datastore/mysql/software.go | 30 ++++ server/datastore/mysql/software_test.go | 152 +++++++++++++++- server/fleet/datastore.go | 4 + server/fleet/service.go | 4 + server/fleet/software_installer.go | 16 ++ server/mock/datastore_mock.go | 12 ++ server/service/handler.go | 1 + server/service/integration_enterprise_test.go | 163 ++++++++++++++++++ server/service/orbit.go | 48 ++++++ 10 files changed, 424 insertions(+), 7 deletions(-) create mode 100644 changes/18675-add-orbit-endpoint-for-software-install-results diff --git a/changes/18675-add-orbit-endpoint-for-software-install-results b/changes/18675-add-orbit-endpoint-for-software-install-results new file mode 100644 index 0000000000..5cc376d760 --- /dev/null +++ b/changes/18675-add-orbit-endpoint-for-software-install-results @@ -0,0 +1 @@ +* Added the `POST /api/fleet/orbit/software_install/result` endpoint for fleetd to send results for a software installation attempt. diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 72f2b9b2c4..0e6a02b899 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -2024,3 +2024,33 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, hostID uint, includeA } return software, metaData, nil } + +func (ds *Datastore) SetHostSoftwareInstallResult(ctx context.Context, result *fleet.HostSoftwareInstallResultPayload) error { + const stmt = ` + UPDATE + host_software_installs + SET + pre_install_query_output = ?, + install_script_exit_code = ?, + install_script_output = ?, + post_install_script_exit_code = ?, + post_install_script_output = ? + WHERE + execution_id = ? +` + res, err := ds.writer(ctx).ExecContext(ctx, stmt, + result.PreInstallConditionOutput, + result.InstallScriptExitCode, + result.InstallScriptOutput, + result.PostInstallScriptExitCode, + result.PostInstallScriptOutput, + result.InstallUUID, + ) + if err != nil { + return ctxerr.Wrap(ctx, err, "update host software installation result") + } + if n, _ := res.RowsAffected(); n == 0 { + return ctxerr.Wrap(ctx, notFound("HostSoftwareInstall").WithName(result.InstallUUID), "host software installation not found") + } + return nil +} diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index 6b0709524b..882ac86a05 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -63,6 +63,7 @@ func TestSoftware(t *testing.T) { {"insertHostSoftwareInstalledPaths", testInsertHostSoftwareInstalledPaths}, {"VerifySoftwareChecksum", testVerifySoftwareChecksum}, {"ListHostSoftware", testListHostSoftware}, + {"SetHostSoftwareInstallResult", testSetHostSoftwareInstallResult}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -3210,14 +3211,14 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { // record a new install request for i1, this time as pending, and mark install request for b (swi1) as failed time.Sleep(time.Second) // ensure the timestamp is later - ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { - // swi1 is now failed - _, err = q.ExecContext(ctx, ` - UPDATE host_software_installs SET install_script_exit_code = 2 WHERE execution_id = 'uuid1'`) - if err != nil { - return err - } + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: host.ID, + InstallUUID: "uuid1", + InstallScriptExitCode: ptr.Int(2), + }) + require.NoError(t, err) + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { // swi3 has a new install request pending _, err = q.ExecContext(ctx, ` INSERT INTO host_software_installs (execution_id, host_id, software_installer_id) @@ -3365,3 +3366,140 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { }) } } + +func testSetHostSoftwareInstallResult(t *testing.T, ds *Datastore) { + ctx := context.Background() + host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) + + // create a software installer and some host install requests + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + installScript := `echo 'foo'` + res, err := q.ExecContext(ctx, `INSERT INTO script_contents (md5_checksum, contents) VALUES (UNHEX(md5(?)), ?)`, installScript, installScript) + if err != nil { + return err + } + scriptContentID, _ := res.LastInsertId() + + res, err = q.ExecContext(ctx, `INSERT INTO software_titles (name, source) VALUES ('foo', 'apps')`) + if err != nil { + return err + } + titleID, _ := res.LastInsertId() + + res, err = q.ExecContext(ctx, ` + INSERT INTO software_installers + (title_id, filename, version, install_script_content_id, storage_id) + VALUES + (?, ?, ?, ?, unhex(?))`, + titleID, "installer.pkg", "v1.0.0", scriptContentID, hex.EncodeToString([]byte("test"))) + if err != nil { + return err + } + id, _ := res.LastInsertId() + + // create some install requests for the host + for i := 0; i < 3; i++ { + _, err = q.ExecContext(ctx, ` + INSERT INTO host_software_installs (execution_id, host_id, software_installer_id) VALUES (?, ?, ?)`, + fmt.Sprintf("uuid%d", i), host.ID, id) + if err != nil { + return err + } + } + return nil + }) + + checkResults := func(want *fleet.HostSoftwareInstallResultPayload) { + type result struct { + HostID uint `db:"host_id"` + InstallUUID string `db:"execution_id"` + PreInstallConditionOutput *string `db:"pre_install_query_output"` + InstallScriptExitCode *int `db:"install_script_exit_code"` + InstallScriptOutput *string `db:"install_script_output"` + PostInstallScriptExitCode *int `db:"post_install_script_exit_code"` + PostInstallScriptOutput *string `db:"post_install_script_output"` + } + var got result + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(ctx, q, &got, + `SELECT + host_id, + execution_id, + pre_install_query_output, + install_script_exit_code, + install_script_output, + post_install_script_exit_code, + post_install_script_output + FROM + host_software_installs + WHERE execution_id = ?`, want.InstallUUID) + }) + assert.Equal(t, want.HostID, got.HostID) + assert.Equal(t, want.InstallUUID, got.InstallUUID) + if want.PreInstallConditionOutput == nil { + assert.Nil(t, got.PreInstallConditionOutput) + } else { + assert.NotNil(t, got.PreInstallConditionOutput) + assert.Equal(t, *want.PreInstallConditionOutput, *got.PreInstallConditionOutput) + } + assert.Equal(t, want.InstallScriptExitCode, got.InstallScriptExitCode) + if want.InstallScriptOutput == nil { + assert.Nil(t, got.InstallScriptOutput) + } else { + assert.NotNil(t, got.InstallScriptOutput) + assert.EqualValues(t, want.InstallScriptOutput, got.InstallScriptOutput) + } + assert.Equal(t, want.PostInstallScriptExitCode, got.PostInstallScriptExitCode) + if want.PostInstallScriptOutput == nil { + assert.Nil(t, got.PostInstallScriptOutput) + } else { + assert.NotNil(t, got.PostInstallScriptOutput) + assert.EqualValues(t, want.InstallScriptOutput, got.InstallScriptOutput) + } + } + + // set a result with all fields provided + want := &fleet.HostSoftwareInstallResultPayload{ + HostID: host.ID, + InstallUUID: "uuid0", + PreInstallConditionOutput: ptr.String("1"), + InstallScriptExitCode: ptr.Int(0), + InstallScriptOutput: ptr.String("ok"), + PostInstallScriptExitCode: ptr.Int(0), + PostInstallScriptOutput: ptr.String("ok"), + } + err := ds.SetHostSoftwareInstallResult(ctx, want) + require.NoError(t, err) + checkResults(want) + + // set a result with only the pre-condition that failed + want = &fleet.HostSoftwareInstallResultPayload{ + HostID: host.ID, + InstallUUID: "uuid1", + PreInstallConditionOutput: ptr.String(""), + } + err = ds.SetHostSoftwareInstallResult(ctx, want) + require.NoError(t, err) + checkResults(want) + + // set a result with only the install that failed + want = &fleet.HostSoftwareInstallResultPayload{ + HostID: host.ID, + InstallUUID: "uuid2", + InstallScriptExitCode: ptr.Int(1), + InstallScriptOutput: ptr.String("fail"), + } + err = ds.SetHostSoftwareInstallResult(ctx, want) + require.NoError(t, err) + checkResults(want) + + // set a result for a non-existing uuid + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: host.ID, + InstallUUID: "uuid-no-such", + InstallScriptExitCode: ptr.Int(0), + InstallScriptOutput: ptr.String("ok"), + }) + require.Error(t, err) + require.True(t, fleet.IsNotFound(err)) +} diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 93792dcfee..ceb8dd3649 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -549,6 +549,10 @@ type Datastore interface { ListHostSoftware(ctx context.Context, hostID uint, includeAvailableForInstall bool, opts ListOptions) ([]*HostSoftwareWithInstaller, *PaginationMetadata, error) + // SetHostSoftwareInstallResult records the result of a software installation + // attempt on the host. + SetHostSoftwareInstallResult(ctx context.Context, result *HostSoftwareInstallResultPayload) error + /////////////////////////////////////////////////////////////////////////////// // OperatingSystemsStore diff --git a/server/fleet/service.go b/server/fleet/service.go index b90bab4941..cc581bb5ac 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -619,6 +619,10 @@ type Service interface { SoftwareByID(ctx context.Context, id uint, teamID *uint, includeCVEScores bool) (*Software, error) CountSoftware(ctx context.Context, opt SoftwareListOptions) (int, error) + // SaveHostSoftwareInstallResult saves information about execution of a + // software installation on a host. + SaveHostSoftwareInstallResult(ctx context.Context, result *HostSoftwareInstallResultPayload) error + // ///////////////////////////////////////////////////////////////////////////// // Software Titles diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index cb72250781..6472309c16 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -191,3 +191,19 @@ type HostSoftwareInstalledVersion struct { Vulnerabilities []string `json:"vulnerabilities" db:"vulnerabilities"` InstalledPaths []string `json:"installed_paths" db:"installed_paths"` } + +// HostSoftwareInstallResultPayload is the payload provided by fleetd to record +// the results of a software installation attempt. +type HostSoftwareInstallResultPayload struct { + HostID uint `json:"host_id"` + InstallUUID string `json:"install_uuid"` + + // the following fields are nil-able because the corresponding steps may not + // have been executed (optional step, or executed conditionally to a previous + // step). + PreInstallConditionOutput *string `json:"pre_install_condition_output"` + InstallScriptExitCode *int `json:"install_script_exit_code"` + InstallScriptOutput *string `json:"install_script_output"` + PostInstallScriptExitCode *int `json:"post_install_script_exit_code"` + PostInstallScriptOutput *string `json:"post_install_script_output"` +} diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 79b2686cf8..332f6cf3e3 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -403,6 +403,8 @@ type ListCVEsFunc func(ctx context.Context, maxAge time.Duration) ([]fleet.CVEMe type ListHostSoftwareFunc func(ctx context.Context, hostID uint, includeAvailableForInstall bool, opts fleet.ListOptions) ([]*fleet.HostSoftwareWithInstaller, *fleet.PaginationMetadata, error) +type SetHostSoftwareInstallResultFunc func(ctx context.Context, result *fleet.HostSoftwareInstallResultPayload) error + type GetHostOperatingSystemFunc func(ctx context.Context, hostID uint) (*fleet.OperatingSystem, error) type ListOperatingSystemsFunc func(ctx context.Context) ([]fleet.OperatingSystem, error) @@ -1506,6 +1508,9 @@ type DataStore struct { ListHostSoftwareFunc ListHostSoftwareFunc ListHostSoftwareFuncInvoked bool + SetHostSoftwareInstallResultFunc SetHostSoftwareInstallResultFunc + SetHostSoftwareInstallResultFuncInvoked bool + GetHostOperatingSystemFunc GetHostOperatingSystemFunc GetHostOperatingSystemFuncInvoked bool @@ -3642,6 +3647,13 @@ func (s *DataStore) ListHostSoftware(ctx context.Context, hostID uint, includeAv return s.ListHostSoftwareFunc(ctx, hostID, includeAvailableForInstall, opts) } +func (s *DataStore) SetHostSoftwareInstallResult(ctx context.Context, result *fleet.HostSoftwareInstallResultPayload) error { + s.mu.Lock() + s.SetHostSoftwareInstallResultFuncInvoked = true + s.mu.Unlock() + return s.SetHostSoftwareInstallResultFunc(ctx, result) +} + func (s *DataStore) GetHostOperatingSystem(ctx context.Context, hostID uint) (*fleet.OperatingSystem, error) { s.mu.Lock() s.GetHostOperatingSystemFuncInvoked = true diff --git a/server/service/handler.go b/server/service/handler.go index 52e0f84d4d..e5210ca62f 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -808,6 +808,7 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC oe.POST("/api/fleet/orbit/scripts/request", getOrbitScriptEndpoint, orbitGetScriptRequest{}) oe.POST("/api/fleet/orbit/scripts/result", postOrbitScriptResultEndpoint, orbitPostScriptResultRequest{}) oe.PUT("/api/fleet/orbit/device_mapping", putOrbitDeviceMappingEndpoint, orbitPutDeviceMappingRequest{}) + oe.POST("/api/fleet/orbit/software_install/result", postOrbitSoftwareInstallResultEndpoint, orbitPostSoftwareInstallResultRequest{}) oe.POST("/api/fleet/orbit/software_install/package", orbitDownloadSoftwareInstallerEndpoint, orbitDownloadSoftwareInstallerRequest{}) diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 5b1b9b51f1..fa82983d66 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -5,6 +5,7 @@ import ( "context" "database/sql" "encoding/base64" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -8600,6 +8601,12 @@ func (s *integrationEnterpriseTestSuite) TestListHostSoftware() { ctx := context.Background() t := s.T() + // clean up any software titles from previous tests + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(ctx, `DELETE FROM software_titles`) + return err + }) + token := "good_token" host := createHostAndDeviceToken(t, s.ds, token) @@ -8645,6 +8652,162 @@ func (s *integrationEnterpriseTestSuite) TestListHostSoftware() { // TODO(mna): more advanced integration tests with Software Installers once the APIs are in place. } +func (s *integrationEnterpriseTestSuite) TestHostSoftwareInstallResult() { + ctx := context.Background() + t := s.T() + + host := createOrbitEnrolledHost(t, "linux", "", s.ds) + + // create a software installer and some host install requests + // TODO(mna): replace with API calls once they are available + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + installScript := `echo 'foo'` + res, err := q.ExecContext(ctx, `INSERT INTO script_contents (md5_checksum, contents) VALUES (UNHEX(md5(?)), ?)`, installScript, installScript) + if err != nil { + return err + } + scriptContentID, _ := res.LastInsertId() + + res, err = q.ExecContext(ctx, `INSERT INTO software_titles (name, source) VALUES ('foo', 'apps')`) + if err != nil { + return err + } + titleID, _ := res.LastInsertId() + + res, err = q.ExecContext(ctx, ` + INSERT INTO software_installers + (title_id, filename, version, install_script_content_id, storage_id) + VALUES + (?, ?, ?, ?, unhex(?))`, + titleID, "installer.pkg", "v1.0.0", scriptContentID, hex.EncodeToString([]byte("test"))) + if err != nil { + return err + } + id, _ := res.LastInsertId() + + // create some install requests for the host + for i := 0; i < 3; i++ { + _, err = q.ExecContext(ctx, ` + INSERT INTO host_software_installs (execution_id, host_id, software_installer_id) VALUES (?, ?, ?)`, + fmt.Sprintf("uuid%d", i), host.ID, id) + if err != nil { + return err + } + } + return nil + }) + + // TODO(mna): replace with API calls once they are available + type result struct { + HostID uint `db:"host_id"` + InstallUUID string `db:"execution_id"` + PreInstallConditionOutput *string `db:"pre_install_query_output"` + InstallScriptExitCode *int `db:"install_script_exit_code"` + InstallScriptOutput *string `db:"install_script_output"` + PostInstallScriptExitCode *int `db:"post_install_script_exit_code"` + PostInstallScriptOutput *string `db:"post_install_script_output"` + } + checkResults := func(want result) { + var got result + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(ctx, q, &got, + `SELECT + host_id, + execution_id, + pre_install_query_output, + install_script_exit_code, + install_script_output, + post_install_script_exit_code, + post_install_script_output + FROM + host_software_installs + WHERE execution_id = ?`, want.InstallUUID) + }) + assert.Equal(t, want.HostID, got.HostID) + assert.Equal(t, want.InstallUUID, got.InstallUUID) + if want.PreInstallConditionOutput == nil { + assert.Nil(t, got.PreInstallConditionOutput) + } else { + assert.NotNil(t, got.PreInstallConditionOutput) + assert.Equal(t, *want.PreInstallConditionOutput, *got.PreInstallConditionOutput) + } + assert.Equal(t, want.InstallScriptExitCode, got.InstallScriptExitCode) + if want.InstallScriptOutput == nil { + assert.Nil(t, got.InstallScriptOutput) + } else { + assert.NotNil(t, got.InstallScriptOutput) + assert.Equal(t, *want.InstallScriptOutput, *got.InstallScriptOutput) + } + assert.Equal(t, want.PostInstallScriptExitCode, got.PostInstallScriptExitCode) + if want.PostInstallScriptOutput == nil { + assert.Nil(t, got.PostInstallScriptOutput) + } else { + assert.NotNil(t, got.PostInstallScriptOutput) + assert.Equal(t, *want.PostInstallScriptOutput, *got.PostInstallScriptOutput) + } + } + + s.Do("POST", "/api/fleet/orbit/software_install/result", + json.RawMessage(fmt.Sprintf(`{ + "orbit_node_key": %q, + "install_uuid": "uuid0", + "pre_install_condition_output": "1", + "install_script_exit_code": 1, + "install_script_output": "failed" + }`, *host.OrbitNodeKey)), + http.StatusNoContent) + checkResults(result{ + HostID: host.ID, + InstallUUID: "uuid0", + PreInstallConditionOutput: ptr.String("1"), + InstallScriptExitCode: ptr.Int(1), + InstallScriptOutput: ptr.String("failed"), + }) + + s.Do("POST", "/api/fleet/orbit/software_install/result", + json.RawMessage(fmt.Sprintf(`{ + "orbit_node_key": %q, + "install_uuid": "uuid1", + "pre_install_condition_output": "" + }`, *host.OrbitNodeKey)), + http.StatusNoContent) + checkResults(result{ + HostID: host.ID, + InstallUUID: "uuid1", + PreInstallConditionOutput: ptr.String(""), + }) + + s.Do("POST", "/api/fleet/orbit/software_install/result", + json.RawMessage(fmt.Sprintf(`{ + "orbit_node_key": %q, + "install_uuid": "uuid2", + "pre_install_condition_output": "1", + "install_script_exit_code": 0, + "install_script_output": "success", + "post_install_script_exit_code": 1, + "post_install_script_output": "failed" + }`, *host.OrbitNodeKey)), + http.StatusNoContent) + checkResults(result{ + HostID: host.ID, + InstallUUID: "uuid2", + PreInstallConditionOutput: ptr.String("1"), + InstallScriptExitCode: ptr.Int(0), + InstallScriptOutput: ptr.String("success"), + PostInstallScriptExitCode: ptr.Int(1), + PostInstallScriptOutput: ptr.String("failed"), + }) + + // non-existing installation uuid + s.Do("POST", "/api/fleet/orbit/software_install/result", + json.RawMessage(fmt.Sprintf(`{ + "orbit_node_key": %q, + "install_uuid": "uuid-no-such", + "pre_install_condition_output": "" + }`, *host.OrbitNodeKey)), + http.StatusNotFound) +} + func genDistributedReqWithPolicyResults(host *fleet.Host, policyResults map[uint]*bool) submitDistributedQueryResultsRequestShim { var ( results = make(map[string]json.RawMessage) diff --git a/server/service/orbit.go b/server/service/orbit.go index e13bfd3b1b..23522c0ea7 100644 --- a/server/service/orbit.go +++ b/server/service/orbit.go @@ -790,3 +790,51 @@ func (svc *Service) OrbitDownloadSoftwareInstaller(ctx context.Context, installe return nil, fleet.ErrMissingLicense } + +///////////////////////////////////////////////////////////////////////////////// +// Post Orbit software install result +///////////////////////////////////////////////////////////////////////////////// + +type orbitPostSoftwareInstallResultRequest struct { + OrbitNodeKey string `json:"orbit_node_key"` + *fleet.HostSoftwareInstallResultPayload +} + +// interface implementation required by the OrbitClient +func (r *orbitPostSoftwareInstallResultRequest) setOrbitNodeKey(nodeKey string) { + r.OrbitNodeKey = nodeKey +} + +func (r *orbitPostSoftwareInstallResultRequest) orbitHostNodeKey() string { + return r.OrbitNodeKey +} + +type orbitPostSoftwareInstallResultResponse struct { + Err error `json:"error,omitempty"` +} + +func (r orbitPostSoftwareInstallResultResponse) error() error { return r.Err } +func (r orbitPostSoftwareInstallResultResponse) Status() int { return http.StatusNoContent } + +func postOrbitSoftwareInstallResultEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { + req := request.(*orbitPostSoftwareInstallResultRequest) + if err := svc.SaveHostSoftwareInstallResult(ctx, req.HostSoftwareInstallResultPayload); err != nil { + return orbitPostSoftwareInstallResultResponse{Err: err}, nil + } + return orbitPostSoftwareInstallResultResponse{}, nil +} + +func (svc *Service) SaveHostSoftwareInstallResult(ctx context.Context, result *fleet.HostSoftwareInstallResultPayload) error { + // this is not a user-authenticated endpoint + svc.authz.SkipAuthorization(ctx) + + host, ok := hostctx.FromContext(ctx) + if !ok { + return newOsqueryError("internal error: missing host from request context") + } + + // always use the authenticated host's ID as host_id + result.HostID = host.ID + err := svc.ds.SetHostSoftwareInstallResult(ctx, result) + return ctxerr.Wrap(ctx, err, "save host software installation result") +} From 4c99ebebaf03d52622286b7e27ca4f2c79a293f0 Mon Sep 17 00:00:00 2001 From: Gabriel Hernandez Date: Mon, 6 May 2024 13:49:49 +0100 Subject: [PATCH 16/56] UI updates to software page to support added software feature. (#18731) relates to #18328 make updates to the software titles page to support new add software feature. this includes. **Change of page description** ![image](https://github.com/fleetdm/fleet/assets/1153709/e90a2149-54c4-41f0-a1ec-12ebc4619d6c) **new install status column and change order of `Type` and `verison` columns** ![image](https://github.com/fleetdm/fleet/assets/1153709/662841fd-2f9e-489c-adc3-fbf1442228b2) **adding new dropdown filter option and conditionally showing it for titles and versions tables** ![image](https://github.com/fleetdm/fleet/assets/1153709/8e81680e-d733-4d63-94b6-b4441cb708e3) - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [x] Manual QA for all new/changed functionality --- ...-updates-to-software-page-for-add-software | 1 + frontend/__mocks__/softwareMock.ts | 1 + frontend/components/icons/Download.tsx | 1 + frontend/components/icons/Install.tsx | 29 +++++ frontend/components/icons/index.ts | 2 + frontend/interfaces/software.ts | 1 + .../DashboardPage/cards/Software/Software.tsx | 4 +- .../SoftwareOSTable/SoftwareOSTable.tsx | 6 +- frontend/pages/SoftwarePage/SoftwarePage.tsx | 31 +++-- .../SoftwareTable/SoftwareTable.tsx | 108 +++++++++++------- .../SoftwareTitlesTableConfig.tsx | 23 +++- .../SoftwareVersionsTableConfig.tsx | 16 +-- .../SoftwareTitles/SoftwareTable/helpers.ts | 30 +++++ .../SoftwareTitles/SoftwareTitles.tsx | 47 +++++--- .../SoftwareVulnerabilitiesTable.tsx | 6 +- .../EmptySoftwareTable/EmptySoftwareTable.tsx | 41 ++++--- .../components/IconCell/IconCell.tsx | 44 +++++++ .../components/IconCell/_styles.scss | 3 + .../SoftwarePage/components/IconCell/index.ts | 1 + .../hosts/details/cards/Software/Software.tsx | 28 +---- frontend/services/entities/software.ts | 7 +- frontend/utilities/constants.tsx | 16 --- 22 files changed, 286 insertions(+), 160 deletions(-) create mode 100644 changes/issue-18328-updates-to-software-page-for-add-software create mode 100644 frontend/components/icons/Install.tsx create mode 100644 frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/helpers.ts create mode 100644 frontend/pages/SoftwarePage/components/IconCell/IconCell.tsx create mode 100644 frontend/pages/SoftwarePage/components/IconCell/_styles.scss create mode 100644 frontend/pages/SoftwarePage/components/IconCell/index.ts diff --git a/changes/issue-18328-updates-to-software-page-for-add-software b/changes/issue-18328-updates-to-software-page-for-add-software new file mode 100644 index 0000000000..b3b9a84add --- /dev/null +++ b/changes/issue-18328-updates-to-software-page-for-add-software @@ -0,0 +1 @@ +- udpates software page to support new add software feature. diff --git a/frontend/__mocks__/softwareMock.ts b/frontend/__mocks__/softwareMock.ts index 6ad48d7640..663818cf26 100644 --- a/frontend/__mocks__/softwareMock.ts +++ b/frontend/__mocks__/softwareMock.ts @@ -45,6 +45,7 @@ export const createMockSoftwareTitleVersion = ( const DEFAULT_SOFTWARE_TITLE_MOCK: ISoftwareTitle = { id: 1, name: "mock software 1.app", + software_package: null, versions_count: 1, source: "apps", hosts_count: 1, diff --git a/frontend/components/icons/Download.tsx b/frontend/components/icons/Download.tsx index dedd7ebe3d..96b4683c2c 100644 --- a/frontend/components/icons/Download.tsx +++ b/frontend/components/icons/Download.tsx @@ -6,6 +6,7 @@ interface IDownload { color?: Colors; size?: IconSizes; } + const Download = ({ color = "ui-fleet-black-75", size = "medium", diff --git a/frontend/components/icons/Install.tsx b/frontend/components/icons/Install.tsx new file mode 100644 index 0000000000..768589d0c6 --- /dev/null +++ b/frontend/components/icons/Install.tsx @@ -0,0 +1,29 @@ +import React from "react"; +import { COLORS, Colors } from "styles/var/colors"; + +interface IInstallProps { + color?: Colors; +} + +const Install = ({ color = "ui-fleet-black-50" }: IInstallProps) => { + return ( + + + + + + + + + + + + ); +}; + +export default Install; diff --git a/frontend/components/icons/index.ts b/frontend/components/icons/index.ts index f2bee5bd92..09cd21f479 100644 --- a/frontend/components/icons/index.ts +++ b/frontend/components/icons/index.ts @@ -53,6 +53,7 @@ import Profile from "./Profile"; import Download from "./Download"; import Upload from "./Upload"; import Refresh from "./Refresh"; +import Install from "./Install"; // a mapping of the usable names of icons to the icon source. export const ICON_MAP = { @@ -110,6 +111,7 @@ export const ICON_MAP = { download: Download, upload: Upload, refresh: Refresh, + install: Install, }; export type IconNames = keyof typeof ICON_MAP; diff --git a/frontend/interfaces/software.ts b/frontend/interfaces/software.ts index 8f47761baf..30705dc094 100644 --- a/frontend/interfaces/software.ts +++ b/frontend/interfaces/software.ts @@ -53,6 +53,7 @@ export interface ISoftwareTitleVersion { export interface ISoftwareTitle { id: number; name: string; + software_package: string | null; versions_count: number; source: string; hosts_count: number; diff --git a/frontend/pages/DashboardPage/cards/Software/Software.tsx b/frontend/pages/DashboardPage/cards/Software/Software.tsx index db4ab39198..629082ecba 100644 --- a/frontend/pages/DashboardPage/cards/Software/Software.tsx +++ b/frontend/pages/DashboardPage/cards/Software/Software.tsx @@ -98,7 +98,6 @@ const Software = ({ emptyComponent={() => ( )} showMarkAllPages={false} @@ -125,8 +124,7 @@ const Software = ({ emptyComponent={() => ( )} showMarkAllPages={false} diff --git a/frontend/pages/SoftwarePage/SoftwareOS/SoftwareOSTable/SoftwareOSTable.tsx b/frontend/pages/SoftwarePage/SoftwareOS/SoftwareOSTable/SoftwareOSTable.tsx index d528abd9fb..1012306043 100644 --- a/frontend/pages/SoftwarePage/SoftwareOS/SoftwareOSTable/SoftwareOSTable.tsx +++ b/frontend/pages/SoftwarePage/SoftwareOS/SoftwareOSTable/SoftwareOSTable.tsx @@ -180,11 +180,7 @@ const SoftwareOSTable = ({ isLoading={isLoading} resultsTitle="items" emptyComponent={() => ( - + )} defaultSortHeader={orderKey} defaultSortDirection={orderDirection} diff --git a/frontend/pages/SoftwarePage/SoftwarePage.tsx b/frontend/pages/SoftwarePage/SoftwarePage.tsx index 89020ca581..4a16bb9598 100644 --- a/frontend/pages/SoftwarePage/SoftwarePage.tsx +++ b/frontend/pages/SoftwarePage/SoftwarePage.tsx @@ -29,6 +29,7 @@ import TabsWrapper from "components/TabsWrapper"; import ManageAutomationsModal from "./components/ManageSoftwareAutomationsModal"; import AddSoftwareModal from "./components/AddSoftwareModal"; +import { ISoftwareDropdownFilterVal } from "./SoftwareTitles/SoftwareTable/helpers"; interface ISoftwareSubNavItem { name: string; @@ -62,6 +63,16 @@ const getTabIndex = (path: string): number => { }); }; +const getSoftwareFilter = ( + vulnerable?: string, + installable?: string +): ISoftwareDropdownFilterVal => { + if (installable === "true") return "installableSoftware"; + return vulnerable && vulnerable === "true" + ? "vulnerableSoftware" + : "allSoftware"; +}; + // default values for query params used on this page if not provided const DEFAULT_SORT_DIRECTION = "desc"; const DEFAULT_SORT_HEADER = "hosts_count"; @@ -93,6 +104,7 @@ interface ISoftwarePageProps { query: { team_id?: string; vulnerable?: string; + available_for_install?: string; exploit?: string; page?: string; query?: string; @@ -135,10 +147,12 @@ const SoftwarePage = ({ children, router, location }: ISoftwarePageProps) => { : DEFAULT_PAGE; // TODO: move these down into the Software Titles component. const query = queryParams && queryParams.query ? queryParams.query : ""; - const showVulnerableSoftware = - queryParams !== undefined && queryParams.vulnerable === "true"; const showExploitedVulnerabilitiesOnly = queryParams !== undefined && queryParams.exploit === "true"; + const softwareFilter = getSoftwareFilter( + queryParams.vulnerable, + queryParams.available_for_install + ); const [showManageAutomationsModal, setShowManageAutomationsModal] = useState( false @@ -332,15 +346,8 @@ const SoftwarePage = ({ children, router, location }: ISoftwarePageProps) => { const renderHeaderDescription = () => { return (

- Search for installed software{" "} - {(isGlobalAdmin || isGlobalMaintainer) && - (!isPremiumTier || !isAnyTeamSelected) && - "and manage automations for detected vulnerabilities (CVEs)"}{" "} - on{" "} - {isPremiumTier && isAnyTeamSelected - ? "all hosts assigned to this team" - : "all of your hosts"} - . + Manage software and search for installed software, OS and + vulnerabilities {isAnyTeamSelected ? "on this team" : "for all hosts"}.

); }; @@ -376,8 +383,8 @@ const SoftwarePage = ({ children, router, location }: ISoftwarePageProps) => { teamId: teamIdForApi, // TODO: move down into the Software Titles component query, - showVulnerableSoftware, showExploitedVulnerabilitiesOnly, + softwareFilter, })} ); diff --git a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTable.tsx b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTable.tsx index 191e964e3e..a772f0b344 100644 --- a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTable.tsx +++ b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTable.tsx @@ -10,10 +10,7 @@ import { Row } from "react-table"; import PATHS from "router/paths"; import { AppContext } from "context/app"; import { getNextLocationPath } from "utilities/helpers"; -import { - GITHUB_NEW_ISSUE_LINK, - VULNERABLE_DROPDOWN_OPTIONS, -} from "utilities/constants"; +import { GITHUB_NEW_ISSUE_LINK } from "utilities/constants"; import { buildQueryStringFromParams } from "utilities/url"; import { ISoftwareTitlesResponse, @@ -33,6 +30,11 @@ import EmptySoftwareTable from "pages/SoftwarePage/components/EmptySoftwareTable import generateTitlesTableConfig from "./SoftwareTitlesTableConfig"; import generateVersionsTableConfig from "./SoftwareVersionsTableConfig"; +import { + ISoftwareDropdownFilterVal, + SOFTWARE_TITLES_DROPDOWN_OPTIONS, + SOFTWARE_VERSIONS_DROPDOWN_OPTIONS, +} from "./helpers"; interface IRowProps extends Row { original: { @@ -58,7 +60,7 @@ interface ISoftwareTableProps { perPage: number; orderDirection: "asc" | "desc"; orderKey: string; - showVulnerableSoftware: boolean; + softwareFilter: ISoftwareDropdownFilterVal; currentPage: number; teamId?: number; isLoading: boolean; @@ -75,13 +77,11 @@ const SoftwareTable = ({ perPage, orderDirection, orderKey, - showVulnerableSoftware, + softwareFilter, currentPage, teamId, isLoading, }: ISoftwareTableProps) => { - const { isSandboxMode, noSandboxHosts } = useContext(AppContext); - const currentPath = showVersions ? PATHS.SOFTWARE_VERSIONS : PATHS.SOFTWARE_TITLES; @@ -96,8 +96,6 @@ const SoftwareTable = ({ return val !== orderDirection; case "sortHeader": return val !== orderKey; - case "vulnerable": - return val !== showVulnerableSoftware.toString(); case "pageIndex": return val !== currentPage; default: @@ -106,21 +104,29 @@ const SoftwareTable = ({ }); return changedEntry?.[0] ?? ""; }, - [currentPage, orderDirection, orderKey, query, showVulnerableSoftware] + [currentPage, orderDirection, orderKey, query] ); const generateNewQueryParams = useCallback( (newTableQuery: ITableQueryData, changedParam: string) => { - return { + const newQueryParam: Record = { query: newTableQuery.searchQuery, team_id: teamId, order_direction: newTableQuery.sortDirection, order_key: newTableQuery.sortHeader, - vulnerable: showVulnerableSoftware.toString(), page: changedParam === "pageIndex" ? newTableQuery.pageIndex : 0, }; + if (softwareFilter === "installableSoftware") { + newQueryParam.available_for_install = true.toString(); + } else { + newQueryParam.vulnerable = ( + softwareFilter === "vulnerableSoftware" + ).toString(); + } + + return newQueryParam; }, - [showVulnerableSoftware, teamId] + [softwareFilter, teamId] ); // NOTE: this is called once on initial render and every time the query changes @@ -131,7 +137,9 @@ const SoftwareTable = ({ const changedParam = determineQueryParamChange(newTableQuery); // if nothing has changed, don't update the route. this can happen when - // this handler is called on the inital render. + // this handler is called on the inital render. Can also happen when + // the filter dropdown is changed. That is handled on the onChange handler + // for the dropdown. if (changedParam === "") return; const newRoute = getNextLocationPath({ @@ -167,7 +175,7 @@ const SoftwareTable = ({ // determines if a user be able to search in the table const searchable = isSoftwareEnabled && - (!!tableData || query !== "" || showVulnerableSoftware); + (!!tableData || query !== "" || softwareFilter === "vulnerableSoftware"); const getItemsCountText = () => { const count = data?.count; @@ -187,37 +195,57 @@ const SoftwareTable = ({ }; const handleShowVersionsToggle = () => { + const queryParams: Record = { + query, + team_id: teamId, + order_direction: orderDirection, + order_key: orderKey, + page: 0, // resets page index + }; + + // if we are currently showing installable titles, we want to switch to + // all software versions. If not, we want to keep the current filter. + if (softwareFilter === "installableSoftware") { + queryParams.vulnerable = "false"; + } else { + queryParams.vulnerable = ( + softwareFilter === "vulnerableSoftware" + ).toString(); + } + router.replace( getNextLocationPath({ pathPrefix: showVersions ? PATHS.SOFTWARE_TITLES : PATHS.SOFTWARE_VERSIONS, routeTemplate: "", - queryParams: { - query, - team_id: teamId, - order_direction: orderDirection, - order_key: orderKey, - vulnerable: showVulnerableSoftware.toString(), - page: 0, // resets page index - }, + queryParams, }) ); }; - const handleVulnFilterDropdownChange = (isFilterVulnerable: string) => { + const handleVulnFilterDropdownChange = ( + value: ISoftwareDropdownFilterVal + ) => { + const queryParams: Record = { + query, + team_id: teamId, + order_direction: orderDirection, + order_key: orderKey, + page: 0, // resets page index + }; + + if (value === "installableSoftware") { + queryParams.available_for_install = true; + } else { + queryParams.vulnerable = value === "vulnerableSoftware"; + } + router.replace( getNextLocationPath({ pathPrefix: currentPath, routeTemplate: "", - queryParams: { - query, - team_id: teamId, - order_direction: orderDirection, - order_key: orderKey, - vulnerable: isFilterVulnerable, - page: 0, // resets page index - }, + queryParams, }) ); }; @@ -255,6 +283,10 @@ const SoftwareTable = ({ }; const renderCustomFilters = () => { + const options = showVersions + ? SOFTWARE_VERSIONS_DROPDOWN_OPTIONS + : SOFTWARE_TITLES_DROPDOWN_OPTIONS; + return (
@@ -267,9 +299,9 @@ const SoftwareTable = ({ />
( )} defaultSortHeader={orderKey} @@ -323,7 +353,7 @@ const SoftwareTable = ({ // additionalQueries serves as a trigger for the useDeepEffect hook // to fire onQueryChange for events happeing outside of // the TableContainer. - additionalQueries={showVulnerableSoftware ? "vulnerable" : ""} + // additionalQueries={softwareFilter} customControl={searchable ? renderCustomFilters : undefined} stackControls renderCount={renderSoftwareCount} diff --git a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTitlesTableConfig.tsx b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTitlesTableConfig.tsx index 16f3ea087e..e0f9cb54c2 100644 --- a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTitlesTableConfig.tsx +++ b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTitlesTableConfig.tsx @@ -16,6 +16,7 @@ import HeaderCell from "components/TableContainer/DataTable/HeaderCell"; import TextCell from "components/TableContainer/DataTable/TextCell"; import LinkCell from "components/TableContainer/DataTable/LinkCell/LinkCell"; import ViewAllHostsLink from "components/ViewAllHostsLink"; +import IconCell from "pages/SoftwarePage/components/IconCell"; import VersionCell from "../../components/VersionCell"; import VulnerabilitiesCell from "../../components/VulnerabilitiesCell"; @@ -26,6 +27,10 @@ import SoftwareIcon from "../../components/icons/SoftwareIcon"; type ISoftwareTitlesTableConfig = Column; type ITableStringCellProps = IStringCellProps; +type ISoftwarePackageCellProps = CellProps< + ISoftwareTitle, + ISoftwareTitle["software_package"] +>; type IVersionsCellProps = CellProps; type IVulnerabilitiesCellProps = IVersionsCellProps; type IHostCountCellProps = CellProps< @@ -94,12 +99,12 @@ const generateTableHeaders = ( sortType: "caseInsensitive", }, { - Header: "Version", + Header: "Install status", disableSortBy: true, - accessor: "versions", - Cell: (cellProps: IVersionsCellProps) => ( - - ), + accessor: "software_package", + Cell: (cellProps: ISoftwarePackageCellProps) => { + return cellProps.cell.value ? : null; + }, }, { Header: "Type", @@ -109,6 +114,14 @@ const generateTableHeaders = ( ), }, + { + Header: "Version", + disableSortBy: true, + accessor: "versions", + Cell: (cellProps: IVersionsCellProps) => ( + + ), + }, // the "vulnerabilities" accessor is used but the data is actually coming // from the version attribute. We do this as we already have a "versions" // attribute used for the "Version" column and we cannot reuse. This is a diff --git a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareVersionsTableConfig.tsx b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareVersionsTableConfig.tsx index 1345ba3078..fb8fde36d9 100644 --- a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareVersionsTableConfig.tsx +++ b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareVersionsTableConfig.tsx @@ -74,14 +74,6 @@ const generateTableHeaders = ( }, sortType: "caseInsensitive", }, - { - Header: "Version", - disableSortBy: true, - accessor: "version", - Cell: (cellProps: ITableStringCellProps) => ( - - ), - }, { Header: "Type", disableSortBy: true, @@ -90,6 +82,14 @@ const generateTableHeaders = ( ), }, + { + Header: "Version", + disableSortBy: true, + accessor: "version", + Cell: (cellProps: ITableStringCellProps) => ( + + ), + }, { Header: "Vulnerabilities", disableSortBy: true, diff --git a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/helpers.ts b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/helpers.ts new file mode 100644 index 0000000000..59fa04a46c --- /dev/null +++ b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/helpers.ts @@ -0,0 +1,30 @@ +export type ISoftwareDropdownFilterVal = + | "allSoftware" + | "vulnerableSoftware" + | "installableSoftware"; + +export const SOFTWARE_VERSIONS_DROPDOWN_OPTIONS = [ + { + disabled: false, + label: "All software", + value: "allSoftware", + helpText: "All software installed on your hosts.", + }, + { + disabled: false, + label: "Vulnerable software", + value: "vulnerableSoftware", + helpText: + "All software installed on your hosts with detected vulnerabilities.", + }, +]; + +export const SOFTWARE_TITLES_DROPDOWN_OPTIONS = [ + ...SOFTWARE_VERSIONS_DROPDOWN_OPTIONS, + { + disabled: false, + label: "Available for install", + value: "installableSoftware", + helpText: "Software that can be installed on your hosts.", + }, +]; diff --git a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTitles.tsx b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTitles.tsx index 18b13e555e..b594420270 100644 --- a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTitles.tsx +++ b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTitles.tsx @@ -1,8 +1,7 @@ -/** +/** software/titles Software tab - software/versions Software tab (version toggle on) + software/versions Software tab (version toggle on) */ - import React from "react"; import { InjectedRouter } from "react-router"; import { useQuery } from "react-query"; @@ -16,7 +15,9 @@ import softwareAPI, { import Spinner from "components/Spinner"; import TableDataError from "components/DataError"; + import SoftwareTable from "./SoftwareTable"; +import { ISoftwareDropdownFilterVal } from "./SoftwareTable/helpers"; const baseClass = "software-titles"; @@ -41,7 +42,7 @@ interface ISoftwareTitlesProps { perPage: number; orderDirection: "asc" | "desc"; orderKey: string; - showVulnerableSoftware: boolean; + softwareFilter: ISoftwareDropdownFilterVal; currentPage: number; teamId?: number; } @@ -53,12 +54,31 @@ const SoftwareTitles = ({ perPage, orderDirection, orderKey, - showVulnerableSoftware, + softwareFilter, currentPage, teamId, }: ISoftwareTitlesProps) => { const showVersions = location.pathname === PATHS.SOFTWARE_VERSIONS; + const generateSoftwareTitlesQueryKey = (): ISoftwareTitlesQueryKey => { + const queryKey: ISoftwareTitlesQueryKey = { + scope: "software-titles", + page: currentPage, + perPage, + query, + orderDirection, + orderKey, + teamId, + }; + if (softwareFilter === "installableSoftware") { + queryKey.availableForInstall = true; + } else { + queryKey.vulnerable = softwareFilter === "vulnerableSoftware"; + } + + return queryKey; + }; + // request to get software data const { data: titlesData, @@ -71,18 +91,7 @@ const SoftwareTitles = ({ ISoftwareTitlesResponse, ISoftwareTitlesQueryKey[] >( - [ - { - scope: "software-titles", - page: currentPage, - perPage, - query, - orderDirection, - orderKey, - teamId, - vulnerable: showVulnerableSoftware, - }, - ], + [generateSoftwareTitlesQueryKey()], ({ queryKey }) => softwareAPI.getSoftwareTitles(queryKey[0]), { ...QUERY_OPTIONS, @@ -111,7 +120,7 @@ const SoftwareTitles = ({ orderDirection, orderKey, teamId, - vulnerable: showVulnerableSoftware, + vulnerable: softwareFilter === "vulnerableSoftware", }, ], ({ queryKey }) => softwareAPI.getSoftwareVersions(queryKey[0]), @@ -140,7 +149,7 @@ const SoftwareTitles = ({ perPage={perPage} orderDirection={orderDirection} orderKey={orderKey} - showVulnerableSoftware={showVulnerableSoftware} + softwareFilter={softwareFilter} currentPage={currentPage} teamId={teamId} isLoading={isTitlesFetching || isVersionsFetching} diff --git a/frontend/pages/SoftwarePage/SoftwareVulnerabilities/SoftwareVulnerabilitiesTable/SoftwareVulnerabilitiesTable.tsx b/frontend/pages/SoftwarePage/SoftwareVulnerabilities/SoftwareVulnerabilitiesTable/SoftwareVulnerabilitiesTable.tsx index b0c07e52d6..6333eb526e 100644 --- a/frontend/pages/SoftwarePage/SoftwareVulnerabilities/SoftwareVulnerabilitiesTable/SoftwareVulnerabilitiesTable.tsx +++ b/frontend/pages/SoftwarePage/SoftwareVulnerabilities/SoftwareVulnerabilitiesTable/SoftwareVulnerabilitiesTable.tsx @@ -248,11 +248,7 @@ const SoftwareVulnerabilitiesTable = ({ isLoading={isLoading} resultsTitle={"items"} emptyComponent={() => ( - + )} defaultSortHeader={orderKey} defaultSortDirection={orderDirection} diff --git a/frontend/pages/SoftwarePage/components/EmptySoftwareTable/EmptySoftwareTable.tsx b/frontend/pages/SoftwarePage/components/EmptySoftwareTable/EmptySoftwareTable.tsx index 1596ea750b..fffeba692d 100644 --- a/frontend/pages/SoftwarePage/components/EmptySoftwareTable/EmptySoftwareTable.tsx +++ b/frontend/pages/SoftwarePage/components/EmptySoftwareTable/EmptySoftwareTable.tsx @@ -6,42 +6,42 @@ import React from "react"; import CustomLink from "components/CustomLink"; import EmptyTable from "components/EmptyTable"; import { IEmptyTableProps } from "interfaces/empty_table"; +import { ISoftwareDropdownFilterVal } from "pages/SoftwarePage/SoftwareTitles/SoftwareTable/helpers"; export interface IEmptySoftwareTableProps { + softwareFilter?: ISoftwareDropdownFilterVal; isSoftwareDisabled?: boolean; - isFilterVulnerable?: boolean; - isSandboxMode?: boolean; isCollectingSoftware?: boolean; isSearching?: boolean; - noSandboxHosts?: boolean; } +const generateTypeText = (softwareFilter?: ISoftwareDropdownFilterVal) => { + if (softwareFilter === "installableSoftware") { + return "installable"; + } + return softwareFilter === "vulnerableSoftware" ? "vulnerable" : ""; +}; + const EmptySoftwareTable = ({ + softwareFilter, isSoftwareDisabled, - isFilterVulnerable, - isSandboxMode, isCollectingSoftware, isSearching, - noSandboxHosts, }: IEmptySoftwareTableProps): JSX.Element => { + const softwareTypeText = generateTypeText(softwareFilter); + const emptySoftware: IEmptyTableProps = { - header: `No ${ - isFilterVulnerable ? "vulnerable " : "" - }software match the current search criteria`, - info: `This report is updated every ${ - isSandboxMode ? "15 minutes" : "hour" - } to protect the performance of your devices.`, + header: `No ${softwareTypeText} software match the current search criteria`, + info: + "This report is updated every hour to protect the performance of your devices.", }; + if (isCollectingSoftware) { emptySoftware.header = "No software detected"; emptySoftware.info = "This report is updated every hour to protect the performance of your devices."; - if (isSandboxMode) { - emptySoftware.info = noSandboxHosts - ? "Fleet begins collecting software inventory after a host is enrolled." - : "Fleet is collecting software inventory"; - } } + if (isSoftwareDisabled) { emptySoftware.header = "Software inventory disabled"; emptySoftware.info = ( @@ -56,11 +56,10 @@ const EmptySoftwareTable = ({ ); } - if (isFilterVulnerable && !isSearching) { + if (softwareFilter === "vulnerableSoftware" && !isSearching) { emptySoftware.header = "No vulnerable software detected"; - emptySoftware.info = `This report is updated every ${ - isSandboxMode ? "15 minutes" : "hour" - } to protect the performance of your devices.`; + emptySoftware.info = + "This report is updated every hour to protect the performance of your devices."; } return ( diff --git a/frontend/pages/SoftwarePage/components/IconCell/IconCell.tsx b/frontend/pages/SoftwarePage/components/IconCell/IconCell.tsx new file mode 100644 index 0000000000..dea3b66b61 --- /dev/null +++ b/frontend/pages/SoftwarePage/components/IconCell/IconCell.tsx @@ -0,0 +1,44 @@ +import React from "react"; +import ReactTooltip from "react-tooltip"; +import { uniqueId } from "lodash"; + +import Icon from "components/Icon"; +import { COLORS } from "styles/var/colors"; +import { IconNames } from "components/icons"; + +const baseClass = "icon-cell"; + +interface IIssueCellProps { + iconName: IconNames; +} + +const IconCell = ({ iconName }: IIssueCellProps) => { + const tooltipID = uniqueId(); + + return ( +
+ + + + + + {/* TODO: enhance to be dynmaic */} + Software can be installed on Host details page. + + +
+ ); +}; + +export default IconCell; diff --git a/frontend/pages/SoftwarePage/components/IconCell/_styles.scss b/frontend/pages/SoftwarePage/components/IconCell/_styles.scss new file mode 100644 index 0000000000..149d08840d --- /dev/null +++ b/frontend/pages/SoftwarePage/components/IconCell/_styles.scss @@ -0,0 +1,3 @@ +.icon-cell { + text-align: center; +} diff --git a/frontend/pages/SoftwarePage/components/IconCell/index.ts b/frontend/pages/SoftwarePage/components/IconCell/index.ts new file mode 100644 index 0000000000..4406703d62 --- /dev/null +++ b/frontend/pages/SoftwarePage/components/IconCell/index.ts @@ -0,0 +1 @@ +export { default } from "./IconCell"; diff --git a/frontend/pages/hosts/details/cards/Software/Software.tsx b/frontend/pages/hosts/details/cards/Software/Software.tsx index 74f7305585..b05c951d01 100644 --- a/frontend/pages/hosts/details/cards/Software/Software.tsx +++ b/frontend/pages/hosts/details/cards/Software/Software.tsx @@ -6,11 +6,8 @@ import { isEmpty } from "lodash"; import { AppContext } from "context/app"; import { ISoftware } from "interfaces/software"; -import { VULNERABLE_DROPDOWN_OPTIONS } from "utilities/constants"; import { buildQueryStringFromParams } from "utilities/url"; -// @ts-ignore -import Dropdown from "components/forms/fields/Dropdown"; import TableContainer from "components/TableContainer"; import { ITableQueryData } from "components/TableContainer/TableContainer"; import Card from "components/Card"; @@ -193,19 +190,6 @@ const SoftwareTable = ({ router.push(path); }; - const renderVulnFilterDropdown = () => { - return ( - - ); - }; - return ( ( - + )} showMarkAllPages={false} isAllPagesSelected={false} searchable - customControl={renderVulnFilterDropdown} isClientSidePagination onClientSidePaginationChange={onClientSidePaginationChange} isClientSideFilter @@ -262,10 +241,7 @@ const SoftwareTable = ({ )} ) : ( - + )} ); diff --git a/frontend/services/entities/software.ts b/frontend/services/entities/software.ts index 3cadcdfd3d..725370636b 100644 --- a/frontend/services/entities/software.ts +++ b/frontend/services/entities/software.ts @@ -19,6 +19,7 @@ export interface ISoftwareApiParams { orderDirection?: "asc" | "desc"; query?: string; vulnerable?: boolean; + availableForInstall?: boolean; teamId?: number; } @@ -101,6 +102,7 @@ export default { orderDirection: orderDir = ORDER_DIRECTION, query, vulnerable, + availableForInstall, teamId, }: ISoftwareApiParams): Promise => { const { SOFTWARE } = endpoints; @@ -112,6 +114,7 @@ export default { teamId, query, vulnerable, + availableForInstall, }; const snakeCaseParams = convertParamsToSnakeCase(queryParams); @@ -155,7 +158,9 @@ export default { return sendRequest("GET", path); }, - getSoftwareTitles: (params: ISoftwareApiParams) => { + getSoftwareTitles: ( + params: ISoftwareApiParams + ): Promise => { const { SOFTWARE_TITLES } = endpoints; const snakeCaseParams = convertParamsToSnakeCase(params); const queryString = buildQueryStringFromParams(snakeCaseParams); diff --git a/frontend/utilities/constants.tsx b/frontend/utilities/constants.tsx index b3aaca9503..9480a1432a 100644 --- a/frontend/utilities/constants.tsx +++ b/frontend/utilities/constants.tsx @@ -317,22 +317,6 @@ export const HOSTS_SEARCH_BOX_PLACEHOLDER = export const HOSTS_SEARCH_BOX_TOOLTIP = "Search hosts by name, hostname, UUID, serial number, or private IP address"; -export const VULNERABLE_DROPDOWN_OPTIONS = [ - { - disabled: false, - label: "All software", - value: false, - helpText: "All software installed on your hosts.", - }, - { - disabled: false, - label: "Vulnerable software", - value: true, - helpText: - "All software installed on your hosts with detected vulnerabilities.", - }, -]; - export const EXPLOITED_VULNERABILITIES_DROPDOWN_OPTIONS = [ { disabled: false, From bd3c0a1e9ae95e3561f15d8da88dbd7b723757d4 Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Mon, 6 May 2024 11:41:31 -0300 Subject: [PATCH 17/56] adjust logic to get default scripts (#18719) This tweaks the logic to get default install/remove scripts to delegate the variable replacement to `fleetd` --- ee/server/service/software_installers.go | 5 +- frontend/interfaces/software.ts | 4 -- .../utilities/software_install_scripts.ts | 36 +++------- pkg/file/management.go | 69 +++++++------------ pkg/file/management_test.go | 16 ++--- pkg/file/scripts/README.md | 4 +- .../testdata/scripts/install_deb.sh.golden | 2 +- .../testdata/scripts/install_exe.ps1.golden | 2 +- .../testdata/scripts/install_msi.ps1.golden | 2 +- .../testdata/scripts/install_pkg.sh.golden | 2 +- .../testdata/scripts/remove_deb.sh.golden | 2 +- .../testdata/scripts/remove_exe.ps1.golden | 2 +- .../testdata/scripts/remove_msi.ps1.golden | 2 +- .../testdata/scripts/remove_pkg.sh.golden | 2 +- 14 files changed, 51 insertions(+), 99 deletions(-) diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index 39bc31108f..2783909743 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "errors" "net/http" - "path/filepath" "strings" "github.com/fleetdm/fleet/v4/pkg/file" @@ -58,9 +57,7 @@ func (svc *Service) UploadSoftwareInstaller(ctx context.Context, payload *fleet. } if payload.InstallScript == "" { - installerType := file.InstallerType(strings.TrimPrefix(filepath.Ext(payload.Filename), ".")) - installerPath := payload.Filename // TODO: Confirm pending product input - payload.InstallScript = file.GetInstallScript(installerType, installerPath) + payload.InstallScript = file.GetInstallScript(payload.Filename) } // TODO: basic validation of install and post-install script (e.g., supported interpreters)? diff --git a/frontend/interfaces/software.ts b/frontend/interfaces/software.ts index 30705dc094..b96a3f5659 100644 --- a/frontend/interfaces/software.ts +++ b/frontend/interfaces/software.ts @@ -134,7 +134,3 @@ export const formatSoftwareType = ({ } return type; }; - -// ISoftwareInstallerType defines the supported installer types for -// software uploaded by the IT admin. -export type ISoftwareInstallerType = "pkg" | "msi" | "deb" | "exe"; diff --git a/frontend/utilities/software_install_scripts.ts b/frontend/utilities/software_install_scripts.ts index a26fcb0a15..5bc59c6763 100644 --- a/frontend/utilities/software_install_scripts.ts +++ b/frontend/utilities/software_install_scripts.ts @@ -1,5 +1,3 @@ -import { ISoftwareInstallerType } from "interfaces/software"; - // @ts-ignore import installPkg from "../../pkg/file/scripts/install_pkg.sh"; // @ts-ignore @@ -9,42 +7,24 @@ import installExe from "../../pkg/file/scripts/install_exe.ps1"; // @ts-ignore import installDeb from "../../pkg/file/scripts/install_deb.sh"; -const replaceVariables = (rawScript: string, installerPath: string): string => { - return rawScript.replace("$INSTALLER_PATH", installerPath); -}; - /* * getInstallScript returns a string with a script to install the * provided software. - * - * Note that we don't do any sanitization of the arguments here, - * delegating that to the caller which should have the right context - * about what should be escaped. * */ -const getInstallScript = ( - filetype: ISoftwareInstallerType, - path: string -): string => { - let rawScript: string; - switch (filetype) { +const getInstallScript = (fileName: string): string => { + const extension = fileName.split(".").pop(); + switch (extension) { case "pkg": - rawScript = installPkg; - break; + return installPkg; case "msi": - rawScript = installMsi; - break; + return installMsi; case "deb": - rawScript = installDeb; - break; + return installDeb; case "exe": - rawScript = installExe; - break; + return installExe; default: - // this should never happen as this function is type-guarded - throw new Error(`unsupported file type: ${filetype}`); + throw new Error(`unsupported file extension: ${extension}`); } - - return replaceVariables(rawScript, path); }; export default getInstallScript; diff --git a/pkg/file/management.go b/pkg/file/management.go index 28d2e7710a..dfd53b40ec 100644 --- a/pkg/file/management.go +++ b/pkg/file/management.go @@ -2,16 +2,7 @@ package file import ( _ "embed" - "strings" -) - -type InstallerType string - -const ( - InstallerTypeMsi InstallerType = "msi" - InstallerTypeDeb InstallerType = "deb" - InstallerTypePkg InstallerType = "pkg" - InstallerTypeExe InstallerType = "exe" + "path/filepath" ) //go:embed scripts/install_pkg.sh @@ -26,25 +17,20 @@ var installExeScript string //go:embed scripts/install_deb.sh var installDebScript string -// GetInstallScript returns a script that can be used to install the given -// installer based on the provided type -func GetInstallScript(installerType InstallerType, installerPath string) string { - var rawScript string - - switch installerType { - case InstallerTypeMsi: - rawScript = installMsiScript - case InstallerTypeDeb: - rawScript = installDebScript - case InstallerTypePkg: - rawScript = installPkgScript - case InstallerTypeExe: - rawScript = installExeScript +// GetInstallScript returns a script that can be used to install the given file +func GetInstallScript(filename string) string { + switch ext := filepath.Ext(filename); ext { + case ".msi": + return installMsiScript + case ".deb": + return installDebScript + case ".pkg": + return installPkgScript + case ".exe": + return installExeScript default: return "" } - - return replaceVars(rawScript, installerPath) } //go:embed scripts/remove_exe.ps1 @@ -59,27 +45,18 @@ var removeMsiScript string //go:embed scripts/remove_deb.sh var removeDebScript string -// GetRemoveScript returns a script that can be used to remove the given -// installer based on the provided type -func GetRemoveScript(installerType InstallerType, installerPath string) string { - var rawScript string - - switch installerType { - case InstallerTypeMsi: - rawScript = removeMsiScript - case InstallerTypeDeb: - rawScript = removeDebScript - case InstallerTypePkg: - rawScript = removePkgScript - case InstallerTypeExe: - rawScript = removeExeScript +// GetRemoveScript returns a script that can be used to remove the given file +func GetRemoveScript(filename string) string { + switch ext := filepath.Ext(filename); ext { + case ".msi": + return removeMsiScript + case ".deb": + return removeDebScript + case ".pkg": + return removePkgScript + case ".exe": + return removeExeScript default: return "" } - - return replaceVars(rawScript, installerPath) -} - -func replaceVars(rawScript string, installerPath string) string { - return strings.Replace(rawScript, "$INSTALLER_PATH", installerPath, -1) } diff --git a/pkg/file/management_test.go b/pkg/file/management_test.go index 1b6af00f56..59f63d1675 100644 --- a/pkg/file/management_test.go +++ b/pkg/file/management_test.go @@ -23,32 +23,32 @@ func TestMain(m *testing.M) { // // go test ./pkg/file/... -update func TestGetInstallAndRemoveScript(t *testing.T) { - scriptsByType := map[InstallerType][2]string{ - InstallerTypeMsi: { + scriptsByType := map[string][2]string{ + "msi": { "./scripts/install_msi.ps1", "./scripts/remove_msi.ps1", }, - InstallerTypePkg: { + "pkg": { "./scripts/install_pkg.sh", "./scripts/remove_pkg.sh", }, - InstallerTypeDeb: { + "deb": { "./scripts/install_deb.sh", "./scripts/remove_deb.sh", }, - InstallerTypeExe: { + "exe": { "./scripts/install_exe.ps1", "./scripts/remove_exe.ps1", }, } for itype, scripts := range scriptsByType { - installerPath := "./foo/bar baz.f" + installerPath := "./foo/bar baz." + itype - gotScript := GetInstallScript(itype, installerPath) + gotScript := GetInstallScript(installerPath) assertGoldenMatches(t, scripts[0], gotScript, *update) - gotScript = GetRemoveScript(itype, installerPath) + gotScript = GetRemoveScript(installerPath) assertGoldenMatches(t, scripts[1], gotScript, *update) } } diff --git a/pkg/file/scripts/README.md b/pkg/file/scripts/README.md index 42c81fe507..606801ce68 100644 --- a/pkg/file/scripts/README.md +++ b/pkg/file/scripts/README.md @@ -9,7 +9,9 @@ Scripts are stored on their own files for two reasons: #### Variables -Because the scripts are shared between Go and JS, the convention is to declare variables using `$VAR_NAME` and document its intended usage here. +The scripts in this folder accept variables like `$VAR_NAME` that will be replaced/populated by `fleetd` when they run. + +Supported variables are: - `$INSTALLER_PATH` path to the installer file. diff --git a/pkg/file/testdata/scripts/install_deb.sh.golden b/pkg/file/testdata/scripts/install_deb.sh.golden index d0c8a10f6b..16d246663e 100644 --- a/pkg/file/testdata/scripts/install_deb.sh.golden +++ b/pkg/file/testdata/scripts/install_deb.sh.golden @@ -1 +1 @@ -apt-get install -f "./foo/bar baz.f" +apt-get install -f "$INSTALLER_PATH" diff --git a/pkg/file/testdata/scripts/install_exe.ps1.golden b/pkg/file/testdata/scripts/install_exe.ps1.golden index a3fa160bcc..4aa91f5b1c 100644 --- a/pkg/file/testdata/scripts/install_exe.ps1.golden +++ b/pkg/file/testdata/scripts/install_exe.ps1.golden @@ -1,4 +1,4 @@ -$exeFilePath = "./foo/bar baz.f" +$exeFilePath = "$INSTALLER_PATH" # extract the name of the executable to use as the sub-directory name $exeName = [System.IO.Path]::GetFileName($exeFilePath) diff --git a/pkg/file/testdata/scripts/install_msi.ps1.golden b/pkg/file/testdata/scripts/install_msi.ps1.golden index 4ffd31f6f9..e4f8d8ca90 100644 --- a/pkg/file/testdata/scripts/install_msi.ps1.golden +++ b/pkg/file/testdata/scripts/install_msi.ps1.golden @@ -1,7 +1,7 @@ $logFile = "${env:TEMP}/fleet-install-software.log" $installProcess = Start-Process msiexec.exe ` - -ArgumentList "/quiet /norestart /lv ${logFile} /i `"./foo/bar baz.f`"" ` + -ArgumentList "/quiet /norestart /lv ${logFile} /i `"$INSTALLER_PATH`"" ` -PassThru -Verb RunAs -Wait Get-Content $logFile -Tail 500 diff --git a/pkg/file/testdata/scripts/install_pkg.sh.golden b/pkg/file/testdata/scripts/install_pkg.sh.golden index b736f14690..783d2941a7 100644 --- a/pkg/file/testdata/scripts/install_pkg.sh.golden +++ b/pkg/file/testdata/scripts/install_pkg.sh.golden @@ -1 +1 @@ -installer -pkg "./foo/bar baz.f" -target / +installer -pkg "$INSTALLER_PATH" -target / diff --git a/pkg/file/testdata/scripts/remove_deb.sh.golden b/pkg/file/testdata/scripts/remove_deb.sh.golden index cdc206a19b..5de7123909 100644 --- a/pkg/file/testdata/scripts/remove_deb.sh.golden +++ b/pkg/file/testdata/scripts/remove_deb.sh.golden @@ -1 +1 @@ -apt-get remove -y $(dpkg -f "./foo/bar baz.f" Package) +apt-get remove -y $(dpkg -f "$INSTALLER_PATH" Package) diff --git a/pkg/file/testdata/scripts/remove_exe.ps1.golden b/pkg/file/testdata/scripts/remove_exe.ps1.golden index 0f2548bd6a..eae826c960 100644 --- a/pkg/file/testdata/scripts/remove_exe.ps1.golden +++ b/pkg/file/testdata/scripts/remove_exe.ps1.golden @@ -1,4 +1,4 @@ -$exeFilePath = "./foo/bar baz.f" +$exeFilePath = "$INSTALLER_PATH" # extract the name of the executable to use as the sub-directory name $exeName = [System.IO.Path]::GetFileName($exeFilePath) diff --git a/pkg/file/testdata/scripts/remove_msi.ps1.golden b/pkg/file/testdata/scripts/remove_msi.ps1.golden index 7ae7d04ec0..899a4c9bec 100644 --- a/pkg/file/testdata/scripts/remove_msi.ps1.golden +++ b/pkg/file/testdata/scripts/remove_msi.ps1.golden @@ -1,7 +1,7 @@ $logFile = "${env:TEMP}/fleet-remove-software.log" $removeProcess = Start-Process msiexec.exe ` - -ArgumentList "/quiet /norestart /lv ${logFile} /x `"./foo/bar baz.f`"" ` + -ArgumentList "/quiet /norestart /lv ${logFile} /x `"$INSTALLER_PATH`"" ` -PassThru -Verb RunAs -Wait Get-Content $logFile -Tail 500 diff --git a/pkg/file/testdata/scripts/remove_pkg.sh.golden b/pkg/file/testdata/scripts/remove_pkg.sh.golden index 73d2e32cb1..84ceb7a3fc 100644 --- a/pkg/file/testdata/scripts/remove_pkg.sh.golden +++ b/pkg/file/testdata/scripts/remove_pkg.sh.golden @@ -1,5 +1,5 @@ # grab the identifier from the first PackageInfo we find. Those are placed in different locations depending on the installer -pkg_id=$(tar xOvf "./foo/bar baz.f" --include='*PackageInfo*' 2>/dev/null | sed -n 's/.*identifier="\([^"]*\)".*/\1/p') +pkg_id=$(tar xOvf "$INSTALLER_PATH" --include='*PackageInfo*' 2>/dev/null | sed -n 's/.*identifier="\([^"]*\)".*/\1/p') # remove all the files and empty directories that were installed pkgutil --files $pkg_id | tr '\n' '\0' | xargs -n 1 -0 rm -d From f85eb0aaffd02993133b1af649ead0542423e036 Mon Sep 17 00:00:00 2001 From: Jahziel Villasana-Espinoza Date: Mon, 6 May 2024 15:09:25 -0400 Subject: [PATCH 18/56] feat: get install results endpoint (#18751) > Related issue: #18335 --- .vscode/settings.json | 3 +- changes/17865-get-install-results | 2 + ee/server/service/software_installers.go | 19 +++ server/authz/policy.rego | 18 +++ server/authz/policy_test.go | 128 ++++++++++++------ server/datastore/mysql/software_installers.go | 52 ++++++- .../mysql/software_installers_test.go | 105 ++++++++++++++ server/fleet/datastore.go | 2 + server/fleet/service.go | 3 + server/fleet/software_installer.go | 6 +- server/mock/datastore_mock.go | 12 ++ server/service/handler.go | 1 + server/service/integration_mdm_test.go | 17 ++- server/service/software_installers.go | 30 ++++ 14 files changed, 354 insertions(+), 44 deletions(-) create mode 100644 changes/17865-get-install-results diff --git a/.vscode/settings.json b/.vscode/settings.json index 09e38123d8..3027908ef5 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -38,5 +38,6 @@ "prettier.requireConfig": true, "yaml.schemas": { "https://json.schemastore.org/codecov.json": ".github/workflows/codecov.yml" - } + }, + "favorites.sortOrder": "ASC" } diff --git a/changes/17865-get-install-results b/changes/17865-get-install-results new file mode 100644 index 0000000000..46a6cdd5bd --- /dev/null +++ b/changes/17865-get-install-results @@ -0,0 +1,2 @@ +- Adds the `/software/install/results/:install_uuid` endpoint, which can be used to get the results + for a software install attempt. \ No newline at end of file diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index 2783909743..f55c3a2555 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -233,3 +233,22 @@ func (svc *Service) InstallSoftwareTitle(ctx context.Context, hostID uint, softw return nil } + +func (svc *Service) GetSoftwareInstallResults(ctx context.Context, resultUUID string) (*fleet.HostSoftwareInstallerResult, error) { + // Basic auth check + if err := svc.authz.Authorize(ctx, &fleet.Host{}, fleet.ActionList); err != nil { + return nil, err + } + + res, err := svc.ds.GetSoftwareInstallResults(ctx, resultUUID) + if err != nil { + return nil, err + } + + // Team specific auth check + if err := svc.authz.Authorize(ctx, &fleet.HostSoftwareInstallerResultAuthz{HostTeamID: res.HostTeamID}, fleet.ActionRead); err != nil { + return nil, err + } + + return res, nil +} diff --git a/server/authz/policy.rego b/server/authz/policy.rego index 9d74fb19d4..1836162200 100644 --- a/server/authz/policy.rego +++ b/server/authz/policy.rego @@ -695,6 +695,24 @@ allow { } +# Global admins and maintainers can read software install results on hosts (not +# gitops as this is not something that relates to fleetctl apply). +allow { + object.type == "host_software_installer_result" + subject.global_role == [admin, maintainer, observer, observer_plus][_] + action == read +} + +# Team admin and maintainers can read software install results on hosts for their +# teams (not gitops as this is not something that relates to fleetctl apply). +allow { + object.type == "host_software_installer_result" + not is_null(object.host_team_id) + team_role(subject, object.host_team_id) == [admin, maintainer, observer, observer_plus][_] + action == read +} + + ## # Apple and Windows MDM ## diff --git a/server/authz/policy_test.go b/server/authz/policy_test.go index ba269e32c9..8f81c482b5 100644 --- a/server/authz/policy_test.go +++ b/server/authz/policy_test.go @@ -598,57 +598,107 @@ func TestAuthorizeSoftwareInstaller(t *testing.T) { func TestAuthorizeHostSoftwareInstallerResult(t *testing.T) { t.Parallel() - noTeamInstallRequest := &fleet.HostSoftwareInstallerResultAuthz{} - team1InstallRequest := &fleet.HostSoftwareInstallerResultAuthz{HostTeamID: ptr.Uint(1)} - team2InstallRequest := &fleet.HostSoftwareInstallerResultAuthz{HostTeamID: ptr.Uint(2)} + noTeamInstallResult := &fleet.HostSoftwareInstallerResultAuthz{} + team1InstallResult := &fleet.HostSoftwareInstallerResultAuthz{HostTeamID: ptr.Uint(1)} + team2InstallResult := &fleet.HostSoftwareInstallerResultAuthz{HostTeamID: ptr.Uint(2)} runTestCases(t, []authTestCase{ - {user: nil, object: noTeamInstallRequest, action: write, allow: false}, - {user: nil, object: team1InstallRequest, action: write, allow: false}, - {user: nil, object: team2InstallRequest, action: write, allow: false}, + // Write permissions + {user: nil, object: noTeamInstallResult, action: write, allow: false}, + {user: nil, object: team1InstallResult, action: write, allow: false}, + {user: nil, object: team2InstallResult, action: write, allow: false}, - {user: test.UserNoRoles, object: noTeamInstallRequest, action: write, allow: false}, - {user: test.UserNoRoles, object: team1InstallRequest, action: write, allow: false}, - {user: test.UserNoRoles, object: team2InstallRequest, action: write, allow: false}, + {user: test.UserNoRoles, object: noTeamInstallResult, action: write, allow: false}, + {user: test.UserNoRoles, object: team1InstallResult, action: write, allow: false}, + {user: test.UserNoRoles, object: team2InstallResult, action: write, allow: false}, - {user: test.UserAdmin, object: noTeamInstallRequest, action: write, allow: true}, - {user: test.UserAdmin, object: team1InstallRequest, action: write, allow: true}, - {user: test.UserAdmin, object: team2InstallRequest, action: write, allow: true}, + {user: test.UserAdmin, object: noTeamInstallResult, action: write, allow: true}, + {user: test.UserAdmin, object: team1InstallResult, action: write, allow: true}, + {user: test.UserAdmin, object: team2InstallResult, action: write, allow: true}, - {user: test.UserMaintainer, object: noTeamInstallRequest, action: write, allow: true}, - {user: test.UserMaintainer, object: team1InstallRequest, action: write, allow: true}, - {user: test.UserMaintainer, object: team2InstallRequest, action: write, allow: true}, + {user: test.UserMaintainer, object: noTeamInstallResult, action: write, allow: true}, + {user: test.UserMaintainer, object: team1InstallResult, action: write, allow: true}, + {user: test.UserMaintainer, object: team2InstallResult, action: write, allow: true}, - {user: test.UserObserver, object: noTeamInstallRequest, action: write, allow: false}, - {user: test.UserObserver, object: team1InstallRequest, action: write, allow: false}, - {user: test.UserObserver, object: team2InstallRequest, action: write, allow: false}, + {user: test.UserObserver, object: noTeamInstallResult, action: write, allow: false}, + {user: test.UserObserver, object: team1InstallResult, action: write, allow: false}, + {user: test.UserObserver, object: team2InstallResult, action: write, allow: false}, - {user: test.UserObserverPlus, object: noTeamInstallRequest, action: write, allow: false}, - {user: test.UserObserverPlus, object: team1InstallRequest, action: write, allow: false}, - {user: test.UserObserverPlus, object: team2InstallRequest, action: write, allow: false}, + {user: test.UserObserverPlus, object: noTeamInstallResult, action: write, allow: false}, + {user: test.UserObserverPlus, object: team1InstallResult, action: write, allow: false}, + {user: test.UserObserverPlus, object: team2InstallResult, action: write, allow: false}, - {user: test.UserGitOps, object: noTeamInstallRequest, action: write, allow: false}, - {user: test.UserGitOps, object: team1InstallRequest, action: write, allow: false}, - {user: test.UserGitOps, object: team2InstallRequest, action: write, allow: false}, + {user: test.UserGitOps, object: noTeamInstallResult, action: write, allow: false}, + {user: test.UserGitOps, object: team1InstallResult, action: write, allow: false}, + {user: test.UserGitOps, object: team2InstallResult, action: write, allow: false}, - {user: test.UserTeamGitOpsTeam1, object: noTeamInstallRequest, action: write, allow: false}, - {user: test.UserTeamGitOpsTeam1, object: team1InstallRequest, action: write, allow: false}, - {user: test.UserTeamGitOpsTeam1, object: team2InstallRequest, action: write, allow: false}, + {user: test.UserTeamGitOpsTeam1, object: noTeamInstallResult, action: write, allow: false}, + {user: test.UserTeamGitOpsTeam1, object: team1InstallResult, action: write, allow: false}, + {user: test.UserTeamGitOpsTeam1, object: team2InstallResult, action: write, allow: false}, - {user: test.UserTeamAdminTeam1, object: noTeamInstallRequest, action: write, allow: false}, - {user: test.UserTeamAdminTeam1, object: team1InstallRequest, action: write, allow: true}, - {user: test.UserTeamAdminTeam1, object: team2InstallRequest, action: write, allow: false}, + {user: test.UserTeamAdminTeam1, object: noTeamInstallResult, action: write, allow: false}, + {user: test.UserTeamAdminTeam1, object: team1InstallResult, action: write, allow: true}, + {user: test.UserTeamAdminTeam1, object: team2InstallResult, action: write, allow: false}, - {user: test.UserTeamMaintainerTeam1, object: noTeamInstallRequest, action: write, allow: false}, - {user: test.UserTeamMaintainerTeam1, object: team1InstallRequest, action: write, allow: true}, - {user: test.UserTeamMaintainerTeam1, object: team2InstallRequest, action: write, allow: false}, + {user: test.UserTeamMaintainerTeam1, object: noTeamInstallResult, action: write, allow: false}, + {user: test.UserTeamMaintainerTeam1, object: team1InstallResult, action: write, allow: true}, + {user: test.UserTeamMaintainerTeam1, object: team2InstallResult, action: write, allow: false}, - {user: test.UserTeamObserverTeam1, object: noTeamInstallRequest, action: write, allow: false}, - {user: test.UserTeamObserverTeam1, object: team1InstallRequest, action: write, allow: false}, - {user: test.UserTeamObserverTeam1, object: team2InstallRequest, action: write, allow: false}, + {user: test.UserTeamObserverTeam1, object: noTeamInstallResult, action: write, allow: false}, + {user: test.UserTeamObserverTeam1, object: team1InstallResult, action: write, allow: false}, + {user: test.UserTeamObserverTeam1, object: team2InstallResult, action: write, allow: false}, - {user: test.UserTeamObserverPlusTeam1, object: noTeamInstallRequest, action: write, allow: false}, - {user: test.UserTeamObserverPlusTeam1, object: team1InstallRequest, action: write, allow: false}, - {user: test.UserTeamObserverPlusTeam1, object: team2InstallRequest, action: write, allow: false}, + {user: test.UserTeamObserverPlusTeam1, object: noTeamInstallResult, action: write, allow: false}, + {user: test.UserTeamObserverPlusTeam1, object: team1InstallResult, action: write, allow: false}, + {user: test.UserTeamObserverPlusTeam1, object: team2InstallResult, action: write, allow: false}, + + // Read permissions + {user: nil, object: noTeamInstallResult, action: read, allow: false}, + {user: nil, object: team1InstallResult, action: read, allow: false}, + {user: nil, object: team2InstallResult, action: read, allow: false}, + + {user: test.UserNoRoles, object: noTeamInstallResult, action: read, allow: false}, + {user: test.UserNoRoles, object: team1InstallResult, action: read, allow: false}, + {user: test.UserNoRoles, object: team2InstallResult, action: read, allow: false}, + + {user: test.UserAdmin, object: noTeamInstallResult, action: read, allow: true}, + {user: test.UserAdmin, object: team1InstallResult, action: read, allow: true}, + {user: test.UserAdmin, object: team2InstallResult, action: read, allow: true}, + + {user: test.UserMaintainer, object: noTeamInstallResult, action: read, allow: true}, + {user: test.UserMaintainer, object: team1InstallResult, action: read, allow: true}, + {user: test.UserMaintainer, object: team2InstallResult, action: read, allow: true}, + + {user: test.UserObserver, object: noTeamInstallResult, action: read, allow: true}, + {user: test.UserObserver, object: team1InstallResult, action: read, allow: true}, + {user: test.UserObserver, object: team2InstallResult, action: read, allow: true}, + + {user: test.UserObserverPlus, object: noTeamInstallResult, action: read, allow: true}, + {user: test.UserObserverPlus, object: team1InstallResult, action: read, allow: true}, + {user: test.UserObserverPlus, object: team2InstallResult, action: read, allow: true}, + + {user: test.UserGitOps, object: noTeamInstallResult, action: read, allow: false}, + {user: test.UserGitOps, object: team1InstallResult, action: read, allow: false}, + {user: test.UserGitOps, object: team2InstallResult, action: read, allow: false}, + + {user: test.UserTeamGitOpsTeam1, object: noTeamInstallResult, action: read, allow: false}, + {user: test.UserTeamGitOpsTeam1, object: team1InstallResult, action: read, allow: false}, + {user: test.UserTeamGitOpsTeam1, object: team2InstallResult, action: read, allow: false}, + + {user: test.UserTeamAdminTeam1, object: noTeamInstallResult, action: read, allow: false}, + {user: test.UserTeamAdminTeam1, object: team1InstallResult, action: read, allow: true}, + {user: test.UserTeamAdminTeam1, object: team2InstallResult, action: read, allow: false}, + + {user: test.UserTeamMaintainerTeam1, object: noTeamInstallResult, action: read, allow: false}, + {user: test.UserTeamMaintainerTeam1, object: team1InstallResult, action: read, allow: true}, + {user: test.UserTeamMaintainerTeam1, object: team2InstallResult, action: read, allow: false}, + + {user: test.UserTeamObserverTeam1, object: noTeamInstallResult, action: read, allow: false}, + {user: test.UserTeamObserverTeam1, object: team1InstallResult, action: read, allow: true}, + {user: test.UserTeamObserverTeam1, object: team2InstallResult, action: read, allow: false}, + + {user: test.UserTeamObserverPlusTeam1, object: noTeamInstallResult, action: read, allow: false}, + {user: test.UserTeamObserverPlusTeam1, object: team1InstallResult, action: read, allow: true}, + {user: test.UserTeamObserverPlusTeam1, object: team2InstallResult, action: read, allow: false}, }) } diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index afa4beb751..551c423522 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -177,10 +177,58 @@ func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID ui } _, err = ds.writer(ctx).ExecContext(ctx, insertStmt, - hostID, uuid.NewString(), - softwareTitleID, + hostID, + installerID, ) return ctxerr.Wrap(ctx, err, "inserting new install software request") } + +func (ds *Datastore) GetSoftwareInstallResults(ctx context.Context, resultsUUID string) (*fleet.HostSoftwareInstallerResult, error) { + query := ` +SELECT + hsi.execution_id AS execution_id, + COALESCE(hsi.pre_install_query_output, '') AS pre_install_query_output, + COALESCE(hsi.post_install_script_output, '') AS post_install_script_output, + COALESCE(hsi.install_script_output, '') AS install_script_output, + hsi.host_id AS host_id, + h.computer_name AS host_display_name, + st.name AS software_title, + st.id AS software_title_id, + COALESCE(CASE + WHEN hsi.post_install_script_exit_code IS NOT NULL AND + hsi.post_install_script_exit_code = 0 THEN ? -- installed + WHEN hsi.post_install_script_exit_code IS NOT NULL AND + hsi.post_install_script_exit_code != 0 THEN ? -- failed + WHEN hsi.install_script_exit_code IS NOT NULL AND + hsi.install_script_exit_code = 0 THEN ? -- installed + WHEN hsi.install_script_exit_code IS NOT NULL AND + hsi.install_script_exit_code != 0 THEN ? -- failed + WHEN hsi.pre_install_query_output IS NOT NULL AND + hsi.pre_install_query_output = '' THEN ? -- failed + WHEN hsi.host_id IS NOT NULL THEN ? -- pending + ELSE NULL -- not installed from Fleet installer + END, '') AS status, + si.filename AS software_package, + h.team_id AS host_team_id +FROM + host_software_installs hsi + JOIN hosts h ON h.id = hsi.host_id + JOIN software_installers si ON si.id = hsi.software_installer_id + JOIN software_titles st ON si.title_id = st.id +WHERE + hsi.execution_id = ? + ` + + var dest fleet.HostSoftwareInstallerResult + err := sqlx.GetContext(ctx, ds.reader(ctx), &dest, query, fleet.SoftwareInstallerInstalled, fleet.SoftwareInstallerFailed, fleet.SoftwareInstallerInstalled, fleet.SoftwareInstallerFailed, fleet.SoftwareInstallerFailed, fleet.SoftwareInstallerPending, resultsUUID) + if err != nil { + if err == sql.ErrNoRows { + return nil, ctxerr.Wrap(ctx, notFound("HostSoftwareInstallerResult"), "get host software installer results") + } + return nil, ctxerr.Wrap(ctx, err, "get host software installer results") + } + + return &dest, nil +} diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go index 5645111406..1ecad3de90 100644 --- a/server/datastore/mysql/software_installers_test.go +++ b/server/datastore/mysql/software_installers_test.go @@ -18,6 +18,7 @@ func TestSoftwareInstallers(t *testing.T) { fn func(t *testing.T, ds *Datastore) }{ {"InsertSoftwareInstallRequest", testInsertSoftwareInstallRequest}, + {"GetSoftwareInstallResults", testGetSoftwareInstallResult}, } for _, c := range cases { @@ -76,3 +77,107 @@ func testInsertSoftwareInstallRequest(t *testing.T, ds *Datastore) { }) } } + +func testGetSoftwareInstallResult(t *testing.T, ds *Datastore) { + ctx := context.Background() + + team, err := ds.NewTeam(ctx, &fleet.Team{Name: "team 2"}) + require.NoError(t, err) + teamID := team.ID + + for _, tc := range []struct { + name string + uuid string + expectedStatus fleet.SoftwareInstallerStatus + postInstallScriptEC *uint + preInstallQueryOutput *string + installScriptEC *uint + postInstallScriptOutput *string + installScriptOutput *string + }{ + { + name: "pending install", + uuid: "pending", + expectedStatus: fleet.SoftwareInstallerPending, + postInstallScriptOutput: ptr.String("post install output"), + installScriptOutput: ptr.String("install output"), + }, + { + name: "failing install post install script", + uuid: "fail_post_install_script", + expectedStatus: fleet.SoftwareInstallerFailed, + postInstallScriptEC: ptr.Uint(1), + postInstallScriptOutput: ptr.String("post install output"), + installScriptOutput: ptr.String("install output"), + }, + { + name: "failing install install script", + uuid: "fail_install_script", + expectedStatus: fleet.SoftwareInstallerFailed, + installScriptEC: ptr.Uint(1), + postInstallScriptOutput: ptr.String("post install output"), + installScriptOutput: ptr.String("install output"), + }, + { + name: "failing install pre install query", + uuid: "fail_pre_install_query", + expectedStatus: fleet.SoftwareInstallerFailed, + preInstallQueryOutput: ptr.String(""), + postInstallScriptOutput: ptr.String("post install output"), + installScriptOutput: ptr.String("install output"), + }, + } { + t.Run(tc.name, func(t *testing.T) { + // create a host and software installer + swFilename := "file_" + tc.name + ".pkg" + installerID, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + Title: "foo" + tc.name, + Source: "bar" + tc.name, + InstallScript: "echo " + tc.name, + TeamID: &teamID, + Filename: swFilename, + }) + require.NoError(t, err) + host, err := ds.NewHost(ctx, &fleet.Host{ + Hostname: "macos-test-" + tc.name, + ComputerName: "macos-test-" + tc.name, + OsqueryHostID: ptr.String("osquery-macos-" + tc.name), + NodeKey: ptr.String("node-key-macos-" + tc.name), + UUID: uuid.NewString(), + Platform: "darwin", + TeamID: &teamID, + }) + require.NoError(t, err) + + // Need to insert manually so we have access to the UUID (it's generated in the DS method) + query := `INSERT INTO host_software_installs (execution_id, host_id, software_installer_id, post_install_script_exit_code, install_script_exit_code, pre_install_query_output, install_script_output, post_install_script_output) VALUES (?,?,?,?,?,?,?,?)` + _, err = ds.writer(ctx).ExecContext(ctx, query, tc.uuid, host.ID, installerID, tc.postInstallScriptEC, tc.installScriptEC, tc.preInstallQueryOutput, tc.installScriptOutput, tc.postInstallScriptOutput) + require.NoError(t, err) + + res, err := ds.GetSoftwareInstallResults(ctx, tc.uuid) + require.NoError(t, err) + + require.Equal(t, tc.uuid, res.InstallUUID) + require.Equal(t, tc.expectedStatus, res.Status) + require.Equal(t, swFilename, res.SoftwarePackage) + require.Equal(t, host.ID, res.HostID) + require.Equal(t, host.DisplayName(), res.HostDisplayName) + expectedPreInstallQueryOutput := "" + if tc.preInstallQueryOutput != nil { + expectedPreInstallQueryOutput = *tc.preInstallQueryOutput + } + require.Equal(t, expectedPreInstallQueryOutput, res.PreInstallQueryOutput) + + expectedPostInstallScriptOutput := "" + if tc.postInstallScriptOutput != nil { + expectedPostInstallScriptOutput = *tc.postInstallScriptOutput + } + require.Equal(t, expectedPostInstallScriptOutput, res.PostInstallScriptOutput) + expectedInstallScriptOutput := "" + if tc.installScriptOutput != nil { + expectedInstallScriptOutput = *tc.installScriptOutput + } + require.Equal(t, expectedInstallScriptOutput, res.Output) + }) + } +} diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index ceb8dd3649..9883dd1182 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1475,6 +1475,8 @@ type Datastore interface { // DeleteSoftwareInstaller deletes the software installer corresponding to the id. DeleteSoftwareInstaller(ctx context.Context, id uint) error + + GetSoftwareInstallResults(ctx context.Context, resultsUUID string) (*HostSoftwareInstallerResult, error) } // MDMAppleStore wraps nanomdm's storage and adds methods to deal with diff --git a/server/fleet/service.go b/server/fleet/service.go index cc581bb5ac..bacdcf55f5 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -632,6 +632,9 @@ type Service interface { // InstallSoftwareTitle installs a software title in the given host. InstallSoftwareTitle(ctx context.Context, hostID uint, softwareTitleID uint) error + // GetSoftwareInstallResults gets the results for a particular software install attempt. + GetSoftwareInstallResults(ctx context.Context, installUUID string) (*HostSoftwareInstallerResult, error) + // ///////////////////////////////////////////////////////////////////////////// // Vulnerabilities diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index 6472309c16..68081a4ec0 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -105,7 +105,8 @@ type HostSoftwareInstallerResult struct { HostDisplayName string `json:"host_display_name" db:"host_display_name"` // Status is the status of the software installer package on the host. Status SoftwareInstallerStatus `json:"status" db:"status"` - // Detail is the detail of the software installer package on the host. + // Detail is the detail of the software installer package on the host. TODO: does this field + // have specific values that should be used? If so, how are they calculated? Detail string `json:"detail" db:"detail"` // Output is the output of the software installer package on the host. Output string `json:"output" db:"install_script_output"` @@ -113,6 +114,9 @@ type HostSoftwareInstallerResult struct { PreInstallQueryOutput string `json:"pre_install_query_output" db:"pre_install_query_output"` // PostInstallScriptOutput is the output of the post-install script on the host. PostInstallScriptOutput string `json:"post_install_script_output" db:"post_install_script_output"` + // HostTeamID is the team ID of the host on which this software install was attempted. This + // field is not sent in the response, it is only used for internal authorization. + HostTeamID *uint `json:"-" db:"host_team_id"` } type HostSoftwareInstallerResultAuthz struct { diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 332f6cf3e3..4d34597c28 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -931,6 +931,8 @@ type GetSoftwareInstallerMetadataFunc func(ctx context.Context, id uint) (*fleet type DeleteSoftwareInstallerFunc func(ctx context.Context, id uint) error +type GetSoftwareInstallResultsFunc func(ctx context.Context, resultsUUID string) (*fleet.HostSoftwareInstallerResult, error) + type DataStore struct { HealthCheckFunc HealthCheckFunc HealthCheckFuncInvoked bool @@ -2300,6 +2302,9 @@ type DataStore struct { DeleteSoftwareInstallerFunc DeleteSoftwareInstallerFunc DeleteSoftwareInstallerFuncInvoked bool + GetSoftwareInstallResultsFunc GetSoftwareInstallResultsFunc + GetSoftwareInstallResultsFuncInvoked bool + mu sync.Mutex } @@ -5494,3 +5499,10 @@ func (s *DataStore) DeleteSoftwareInstaller(ctx context.Context, id uint) error s.mu.Unlock() return s.DeleteSoftwareInstallerFunc(ctx, id) } + +func (s *DataStore) GetSoftwareInstallResults(ctx context.Context, resultsUUID string) (*fleet.HostSoftwareInstallerResult, error) { + s.mu.Lock() + s.GetSoftwareInstallResultsFuncInvoked = true + s.mu.Unlock() + return s.GetSoftwareInstallResultsFunc(ctx, resultsUUID) +} diff --git a/server/service/handler.go b/server/service/handler.go index e5210ca62f..6b0d30b7d6 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -376,6 +376,7 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC ue.GET("/api/_version_/fleet/software/package/{id:[0-9]+}", getSoftwareInstallerEndpoint, getSoftwareInstallerRequest{}) ue.POST("/api/_version_/fleet/software/package", uploadSoftwareInstallerEndpoint, uploadSoftwareInstallerRequest{}) ue.DELETE("/api/_version_/fleet/software/package/{id:[0-9]+}", deleteSoftwareInstallerEndpoint, deleteSoftwareInstallerRequest{}) + ue.GET("/api/_version_/fleet/software/install/results/{install_uuid}", getSoftwareInstallResultsEndpoint, getSoftwareInstallResultsRequest{}) // Vulnerabilities ue.GET("/api/_version_/fleet/vulnerabilities", listVulnerabilitiesEndpoint, listVulnerabilitiesRequest{}) diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 86cbd2e3d8..01a594b967 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -8685,13 +8685,28 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerNewInstallRequest() { } s.uploadSoftwareInstaller(payload, http.StatusOK, "") - // install script request succeds + // install script request succeeds titleID := getTitleID(t, payload.Title, "deb_packages") resp = installSoftwareResponse{} s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/hosts/%d/software/install/%d", h.ID, titleID), nil, http.StatusAccepted, &resp) // TODO(roberto): once we have endpoints to retrieve installers, // request them using the orbit node key + + // Get the results, should be pending + r := getHostSoftwareResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/hosts/%d/software", h.ID), nil, http.StatusOK, &r) + require.Len(t, r.Software, 1) + require.NotNil(t, r.Software[0].LastInstall) + installUUID := r.Software[0].LastInstall.InstallUUID + + gsirr := getSoftwareInstallResultsResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/software/install/results/%s", installUUID), nil, http.StatusOK, &gsirr) + require.NoError(t, gsirr.Err) + require.NotNil(t, gsirr.Results) + results := gsirr.Results + require.Equal(t, installUUID, results.InstallUUID) + require.Equal(t, fleet.SoftwareInstallerPending, results.Status) } func (s *integrationMDMTestSuite) uploadSoftwareInstaller(payload *fleet.UploadSoftwareInstallerPayload, expectedStatus int, expectedError string) { diff --git a/server/service/software_installers.go b/server/service/software_installers.go index 8ff62dbf75..1597eb9ec0 100644 --- a/server/service/software_installers.go +++ b/server/service/software_installers.go @@ -246,3 +246,33 @@ func (svc *Service) InstallSoftwareTitle(ctx context.Context, hostID uint, softw return fleet.ErrMissingLicense } + +type getSoftwareInstallResultsRequest struct { + InstallUUID string `url:"install_uuid"` +} + +type getSoftwareInstallResultsResponse struct { + Err error `json:"error,omitempty"` + Results *fleet.HostSoftwareInstallerResult `json:"results,omitempty"` +} + +func (r getSoftwareInstallResultsResponse) error() error { return r.Err } + +func getSoftwareInstallResultsEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { + req := request.(*getSoftwareInstallResultsRequest) + + results, err := svc.GetSoftwareInstallResults(ctx, req.InstallUUID) + if err != nil { + return getSoftwareInstallResultsResponse{Err: err}, nil + } + + return &getSoftwareInstallResultsResponse{Results: results}, nil +} + +func (svc *Service) GetSoftwareInstallResults(ctx context.Context, resultUUID string) (*fleet.HostSoftwareInstallerResult, error) { + // skipauth: No authorization check needed due to implementation returning + // only license error. + svc.authz.SkipAuthorization(ctx) + + return nil, fleet.ErrMissingLicense +} From 731ee68a293fdf949ff2cdd0126108049ea9cef4 Mon Sep 17 00:00:00 2001 From: Dante Catalfamo <43040593+dantecatalfamo@users.noreply.github.com> Date: Mon, 6 May 2024 15:19:45 -0400 Subject: [PATCH 19/56] Orbit software info endpoint (#18690) #18674 --- server/datastore/mysql/software_installers.go | 69 ++++++++- .../mysql/software_installers_test.go | 142 ++++++++++++++++++ server/fleet/datastore.go | 7 + server/fleet/service.go | 5 + server/fleet/software_installer.go | 18 +++ server/mock/datastore_mock.go | 24 +++ server/service/handler.go | 2 + server/service/orbit.go | 55 +++++++ server/service/orbit_test.go | 42 ++++++ 9 files changed, 357 insertions(+), 7 deletions(-) diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 551c423522..2ef91aeacc 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -10,6 +10,61 @@ import ( "github.com/jmoiron/sqlx" ) +func (ds *Datastore) ListPendingSoftwareInstalls(ctx context.Context, hostID uint) ([]string, error) { + const stmt = ` + SELECT + execution_id + FROM + host_software_installs + WHERE + host_id = ? + AND + install_script_exit_code IS NULL + AND + pre_install_query_output IS NULL + ORDER BY + created_at ASC +` + var results []string + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, stmt, hostID); err != nil { + return nil, ctxerr.Wrap(ctx, err, "list pending software installs") + } + return results, nil +} + +func (ds *Datastore) GetSoftwareInstallDetails(ctx context.Context, executionId string) (*fleet.SoftwareInstallDetails, error) { + const stmt = ` + SELECT + hsi.host_id AS host_id, + hsi.execution_id AS execution_id, + hsi.software_installer_id AS installer_id, + si.pre_install_query AS pre_install_condition, + inst.contents AS install_script, + pisnt.contents AS post_install_script + FROM + host_software_installs hsi + INNER JOIN + software_installers si + ON hsi.software_installer_id = si.id + LEFT OUTER JOIN + script_contents inst + ON inst.id = si.install_script_content_id + LEFT OUTER JOIN + script_contents pisnt + ON pisnt.id = si.post_install_script_content_id + WHERE + hsi.execution_id = ?` + + result := &fleet.SoftwareInstallDetails{} + if err := sqlx.GetContext(ctx, ds.reader(ctx), result, stmt, executionId); err != nil { + if err == sql.ErrNoRows { + return nil, ctxerr.Wrap(ctx, notFound("SoftwareInstallerDetails").WithName(executionId), "get software installer details") + } + return nil, ctxerr.Wrap(ctx, err, "list pending software installs") + } + return result, nil +} + func (ds *Datastore) MatchOrCreateSoftwareInstaller(ctx context.Context, payload *fleet.UploadSoftwareInstallerPayload) (uint, error) { titleID, err := ds.getOrGenerateSoftwareInstallerTitleID(ctx, payload.Title, payload.Source) if err != nil { @@ -38,13 +93,13 @@ func (ds *Datastore) MatchOrCreateSoftwareInstaller(ctx context.Context, payload stmt := ` INSERT INTO software_installers ( team_id, - global_or_team_id, - title_id, + global_or_team_id, + title_id, storage_id, - filename, + filename, version, - install_script_content_id, - pre_install_query, + install_script_content_id, + pre_install_query, post_install_script_content_id ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` @@ -105,9 +160,9 @@ SELECT pre_install_query, post_install_script_content_id, uploaded_at -FROM +FROM software_installers -WHERE +WHERE id = ?` var dest fleet.SoftwareInstaller diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go index 1ecad3de90..a0e9b91cec 100644 --- a/server/datastore/mysql/software_installers_test.go +++ b/server/datastore/mysql/software_installers_test.go @@ -2,11 +2,16 @@ package mysql import ( "context" + "database/sql" "testing" + "time" + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/fleetdm/fleet/v4/server/test" "github.com/google/uuid" + "github.com/jmoiron/sqlx" "github.com/stretchr/testify/require" ) @@ -17,6 +22,7 @@ func TestSoftwareInstallers(t *testing.T) { name string fn func(t *testing.T, ds *Datastore) }{ + {"SoftwareInstallerDetails", testListSoftwareInstallerDetails}, {"InsertSoftwareInstallRequest", testInsertSoftwareInstallRequest}, {"GetSoftwareInstallResults", testGetSoftwareInstallResult}, } @@ -29,6 +35,142 @@ func TestSoftwareInstallers(t *testing.T) { } } +func testListSoftwareInstallerDetails(t *testing.T, ds *Datastore) { + ctx := context.Background() + + host1 := test.NewHost(t, ds, "host1", "1", "host1key", "host1uuid", time.Now()) + host2 := test.NewHost(t, ds, "host2", "2", "host2key", "host2uuid", time.Now()) + + script1, err := insertScriptContents(ctx, "hello", ds.writer(ctx)) + require.NoError(t, err) + script1Id, err := script1.LastInsertId() + require.NoError(t, err) + + script2, err := insertScriptContents(ctx, "world", ds.writer(ctx)) + require.NoError(t, err) + script2Id, err := script2.LastInsertId() + require.NoError(t, err) + + installer1, err := insertSoftwareInstaller(ctx, ds.writer(ctx), "file1", "1.0", "SELECT 1", "storage1", script1Id, script2Id) + require.NoError(t, err) + installer1Id, err := installer1.LastInsertId() + require.NoError(t, err) + + installer2, err := insertSoftwareInstaller(ctx, ds.writer(ctx), "file2", "2.0", "SELECT 2", "storage2", script2Id, script1Id) + require.NoError(t, err) + installer2Id, err := installer2.LastInsertId() + require.NoError(t, err) + + hostInstall1, err := insertHostSoftwareInstalls(ctx, ds.writer(ctx), host1.ID, "exec1", uint(installer1Id)) + require.NoError(t, err) + _ = hostInstall1 + + hostInstall2, err := insertHostSoftwareInstalls(ctx, ds.writer(ctx), host1.ID, "exec2", uint(installer2Id)) + require.NoError(t, err) + _ = hostInstall2 + + hostInstall3, err := insertHostSoftwareInstalls(ctx, ds.writer(ctx), host2.ID, "exec3", uint(installer1Id)) + require.NoError(t, err) + _ = hostInstall3 + + hostInstall4, err := insertHostSoftwareInstalls(ctx, ds.writer(ctx), host2.ID, "exec4", uint(installer2Id)) + require.NoError(t, err) + hostInstall4Id, err := hostInstall4.LastInsertId() + require.NoError(t, err) + + _ = ds.writer(ctx).MustExec("UPDATE host_software_installs SET install_script_exit_code = 0 WHERE id = ?", hostInstall4Id) + + hostInstall5, err := insertHostSoftwareInstalls(ctx, ds.writer(ctx), host2.ID, "exec5", uint(installer2Id)) + require.NoError(t, err) + hostInstall5Id, err := hostInstall5.LastInsertId() + require.NoError(t, err) + + _ = ds.writer(ctx).MustExec("UPDATE host_software_installs SET pre_install_query_output = 'output' WHERE id = ?", hostInstall5Id) + + installDetailsList1, err := ds.ListPendingSoftwareInstalls(ctx, host1.ID) + require.NoError(t, err) + require.Equal(t, 2, len(installDetailsList1)) + + installDetailsList2, err := ds.ListPendingSoftwareInstalls(ctx, host2.ID) + require.NoError(t, err) + require.Equal(t, 1, len(installDetailsList2)) + + require.Contains(t, installDetailsList1, "exec1") + require.Contains(t, installDetailsList1, "exec2") + + require.Contains(t, installDetailsList2, "exec3") + + exec1, err := ds.GetSoftwareInstallDetails(ctx, "exec1") + require.NoError(t, err) + + require.Equal(t, host1.ID, exec1.HostID) + require.Equal(t, "exec1", exec1.ExecutionID) + require.Equal(t, "hello", exec1.InstallScript) + require.Equal(t, "world", exec1.PostInstallScript) + require.Equal(t, uint(installer1Id), exec1.InstallerID) + require.Equal(t, "SELECT 1", exec1.PreInstallCondition) +} + +func insertHostSoftwareInstalls( + ctx context.Context, + tx sqlx.ExtContext, + hostId uint, + executionId string, + softwareInstallerId uint, +) (sql.Result, error) { + stmt := ` + INSERT INTO host_software_installs ( + host_id, + execution_id, + software_installer_id + ) VALUES (?, ?, ?) +` + res, err := tx.ExecContext(ctx, stmt, hostId, executionId, softwareInstallerId) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "inserting host software install") + } + + return res, nil +} + +func insertSoftwareInstaller( + ctx context.Context, + tx sqlx.ExtContext, + filename, + version, + preinstallQuery, + storageId string, + installScriptId, + postInstallScriptId int64, +) (sql.Result, error) { + stmt := ` + INSERT INTO software_installers ( + filename, + version, + pre_install_query, + install_script_content_id, + post_install_script_content_id, + storage_id + ) + VALUES (?, ?, ?, ?, ?, ?) +` + res, err := tx.ExecContext(ctx, + stmt, + filename, + version, + preinstallQuery, + installScriptId, + postInstallScriptId, + storageId, + ) + + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "inserting software installer") + } + + return res, nil +} + func testInsertSoftwareInstallRequest(t *testing.T, ds *Datastore) { ctx := context.Background() diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 9883dd1182..47689e340d 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1467,6 +1467,13 @@ type Datastore interface { // Software installers // + // GetSoftwareInstallDetails returns details required to fetch and + // run software installers + GetSoftwareInstallDetails(ctx context.Context, executionId string) (*SoftwareInstallDetails, error) + // ListPendingSoftwareInstalls returns a list of software + // installer execution IDs that have not yet been run for a given host + ListPendingSoftwareInstalls(ctx context.Context, hostID uint) ([]string, error) + // MatchOrCreateSoftwareInstaller matches or creates a new software installer. MatchOrCreateSoftwareInstaller(ctx context.Context, payload *UploadSoftwareInstallerPayload) (uint, error) diff --git a/server/fleet/service.go b/server/fleet/service.go index bacdcf55f5..f4ef1f5fca 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -670,6 +670,11 @@ type Service interface { GetInstaller(ctx context.Context, installer Installer) (io.ReadCloser, int64, error) CheckInstallerExistence(ctx context.Context, installer Installer) error + //////////////////////////////////////////////////////////////////////////////// + // Software Installers + + GetSoftwareInstallDetails(ctx context.Context, installUUID string) (*SoftwareInstallDetails, error) + // ///////////////////////////////////////////////////////////////////////////// // Apple MDM diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index 68081a4ec0..de0f927989 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -35,6 +35,24 @@ func (FailingSoftwareInstallerStore) Exists(ctx context.Context, installerID str return false, errors.New("software installer store not properly configured") } +// SoftwareInstallDetailsResult contains all of the information +// required for a client to pull in and install software from the fleet server +type SoftwareInstallDetails struct { + // HostID is used for authentication on the backend and should not + // be passed to the client + HostID uint `json:"-" db:"host_id"` + // ExecutionID is a unique identifier for this installation + ExecutionID string `json:"install_id" db:"execution_id"` + // InstallerID is the unique identifier for the software package metadata in Fleet. + InstallerID uint `json:"installer_id" db:"installer_id"` + // PreInstallCondition is the query to run as a condition to installing the software package. + PreInstallCondition string `json:"pre_install_condition" db:"pre_install_condition"` + // InstallScript is the script to run to install the software package. + InstallScript string `json:"install_script" db:"install_script"` + // PostInstallScript is the script to run after installing the software package. + PostInstallScript string `json:"post_install_script" db:"post_install_script"` +} + // SoftwareInstaller represents a software installer package that can be used to install software on // hosts in Fleet. type SoftwareInstaller struct { diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 4d34597c28..fe96f32527 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -925,6 +925,10 @@ type WipeHostViaWindowsMDMFunc func(ctx context.Context, host *fleet.Host, cmd * type UpdateHostLockWipeStatusFromAppleMDMResultFunc func(ctx context.Context, hostUUID string, cmdUUID string, requestType string, succeeded bool) error +type GetSoftwareInstallDetailsFunc func(ctx context.Context, executionId string) (*fleet.SoftwareInstallDetails, error) + +type ListPendingSoftwareInstallsFunc func(ctx context.Context, hostID uint) ([]string, error) + type MatchOrCreateSoftwareInstallerFunc func(ctx context.Context, payload *fleet.UploadSoftwareInstallerPayload) (uint, error) type GetSoftwareInstallerMetadataFunc func(ctx context.Context, id uint) (*fleet.SoftwareInstaller, error) @@ -2293,6 +2297,12 @@ type DataStore struct { UpdateHostLockWipeStatusFromAppleMDMResultFunc UpdateHostLockWipeStatusFromAppleMDMResultFunc UpdateHostLockWipeStatusFromAppleMDMResultFuncInvoked bool + GetSoftwareInstallDetailsFunc GetSoftwareInstallDetailsFunc + GetSoftwareInstallDetailsFuncInvoked bool + + ListPendingSoftwareInstallsFunc ListPendingSoftwareInstallsFunc + ListPendingSoftwareInstallsFuncInvoked bool + MatchOrCreateSoftwareInstallerFunc MatchOrCreateSoftwareInstallerFunc MatchOrCreateSoftwareInstallerFuncInvoked bool @@ -5479,6 +5489,20 @@ func (s *DataStore) UpdateHostLockWipeStatusFromAppleMDMResult(ctx context.Conte return s.UpdateHostLockWipeStatusFromAppleMDMResultFunc(ctx, hostUUID, cmdUUID, requestType, succeeded) } +func (s *DataStore) GetSoftwareInstallDetails(ctx context.Context, executionId string) (*fleet.SoftwareInstallDetails, error) { + s.mu.Lock() + s.GetSoftwareInstallDetailsFuncInvoked = true + s.mu.Unlock() + return s.GetSoftwareInstallDetailsFunc(ctx, executionId) +} + +func (s *DataStore) ListPendingSoftwareInstalls(ctx context.Context, hostID uint) ([]string, error) { + s.mu.Lock() + s.ListPendingSoftwareInstallsFuncInvoked = true + s.mu.Unlock() + return s.ListPendingSoftwareInstallsFunc(ctx, hostID) +} + func (s *DataStore) MatchOrCreateSoftwareInstaller(ctx context.Context, payload *fleet.UploadSoftwareInstallerPayload) (uint, error) { s.mu.Lock() s.MatchOrCreateSoftwareInstallerFuncInvoked = true diff --git a/server/service/handler.go b/server/service/handler.go index 6b0d30b7d6..e37828e1b8 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -813,6 +813,8 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC oe.POST("/api/fleet/orbit/software_install/package", orbitDownloadSoftwareInstallerEndpoint, orbitDownloadSoftwareInstallerRequest{}) + oe.POST("/api/fleet/orbit/software_install/details", getOrbitSoftwareInstallDetails, orbitGetSoftwareInstallRequest{}) + oeWindowsMDM := oe.WithCustomMiddleware(mdmConfiguredMiddleware.VerifyWindowsMDM()) oeWindowsMDM.POST("/api/fleet/orbit/disk_encryption_key", postOrbitDiskEncryptionKeyEndpoint, orbitPostDiskEncryptionKeyRequest{}) diff --git a/server/service/orbit.go b/server/service/orbit.go index 23522c0ea7..645cef2c03 100644 --- a/server/service/orbit.go +++ b/server/service/orbit.go @@ -748,6 +748,19 @@ func (svc *Service) SetOrUpdateDiskEncryptionKey(ctx context.Context, encryption } ///////////////////////////////////////////////////////////////////////////////// +// Get Orbit pending software installations +///////////////////////////////////////////////////////////////////////////////// + +type orbitGetSoftwareInstallRequest struct { + OrbitNodeKey string `json:"orbot_node_key"` + InstallUUID string `json:"install_uuid"` +} + +// interface implementation required by the OrbitClient +func (r *orbitGetSoftwareInstallRequest) setOrbitNodeKey(nodeKey string) { + r.OrbitNodeKey = nodeKey +} + // Download Orbit software installer request ///////////////////////////////////////////////////////////////////////////////// @@ -763,6 +776,48 @@ func (r *orbitDownloadSoftwareInstallerRequest) setOrbitNodeKey(nodeKey string) } // interface implementation required by orbit authentication +func (r *orbitGetSoftwareInstallRequest) orbitHostNodeKey() string { + return r.OrbitNodeKey +} + +type orbitGetSoftwareInstallResponse struct { + Err error `json:"error,omitempty"` + *fleet.SoftwareInstallDetails +} + +func (r orbitGetSoftwareInstallResponse) error() error { return r.Err } + +func getOrbitSoftwareInstallDetails(ctx context.Context, request any, svc fleet.Service) (errorer, error) { + req := request.(*orbitGetSoftwareInstallRequest) + details, err := svc.GetSoftwareInstallDetails(ctx, req.InstallUUID) + if err != nil { + return orbitGetSoftwareInstallResponse{Err: err}, nil + } + + return orbitGetSoftwareInstallResponse{SoftwareInstallDetails: details}, nil +} + +func (svc *Service) GetSoftwareInstallDetails(ctx context.Context, installUUID string) (*fleet.SoftwareInstallDetails, error) { + // this is not a user-authenticated endpoint + svc.authz.SkipAuthorization(ctx) + + host, ok := hostctx.FromContext(ctx) + if !ok { + return nil, fleet.OrbitError{Message: "internal error: missing host from request context"} + } + + details, err := svc.ds.GetSoftwareInstallDetails(ctx, installUUID) + if err != nil { + return nil, err + } + + // ensure it cannot get access to a different host's installers + if details.HostID != host.ID { + return nil, ctxerr.Wrap(ctx, newNotFoundError(), "no installer found for this host") + } + return details, nil +} + func (r *orbitDownloadSoftwareInstallerRequest) orbitHostNodeKey() string { return r.OrbitNodeKey } diff --git a/server/service/orbit_test.go b/server/service/orbit_test.go index 97a203d86b..f0cf515c64 100644 --- a/server/service/orbit_test.go +++ b/server/service/orbit_test.go @@ -315,3 +315,45 @@ func TestGetOrbitConfigNudge(t *testing.T) { require.True(t, ds.GetHostOperatingSystemFuncInvoked) }) } + +func TestGetSoftwareInstallDetails(t *testing.T) { + t.Run("hosts can't get each others installers", func(t *testing.T) { + ds := new(mock.Store) + license := &fleet.LicenseInfo{Tier: fleet.TierPremium} + svc, ctx := newTestService(t, ds, nil, nil, &TestServerOpts{License: license, SkipCreateTestUsers: true}) + + ds.GetSoftwareInstallDetailsFunc = func(ctx context.Context, executionId string) (*fleet.SoftwareInstallDetails, error) { + return &fleet.SoftwareInstallDetails{ + HostID: 1, + }, nil + } + + goodCtx := test.HostContext(ctx, &fleet.Host{ + OsqueryHostID: ptr.String("test"), + ID: 1, + MDMInfo: &fleet.HostMDM{ + IsServer: false, + InstalledFromDep: true, + Enrolled: true, + Name: fleet.WellKnownMDMFleet, + }}) + + badCtx := test.HostContext(ctx, &fleet.Host{ + OsqueryHostID: ptr.String("test"), + ID: 2, + MDMInfo: &fleet.HostMDM{ + IsServer: false, + InstalledFromDep: true, + Enrolled: true, + Name: fleet.WellKnownMDMFleet, + }}) + + d1, err := svc.GetSoftwareInstallDetails(goodCtx, "") + require.NoError(t, err) + require.Equal(t, uint(1), d1.HostID) + + d2, err := svc.GetSoftwareInstallDetails(badCtx, "") + require.Error(t, err) + require.Nil(t, d2) + }) +} From 7bb726ba8ee97d55d339504899b937e6be2283fd Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Tue, 7 May 2024 11:28:16 -0400 Subject: [PATCH 20/56] Create and return upcoming/past host activities for software installs (#18772) --- ...2-add-software-installs-to-host-activities | 1 + docs/Using Fleet/Audit-logs.md | 23 +++ docs/Using Fleet/Understanding-host-vitals.md | 15 +- server/datastore/mysql/activities.go | 98 ++++++++-- server/datastore/mysql/activities_test.go | 175 ++++++++++++++---- ...240424124712_AddSoftwareInstallerTables.go | 10 +- server/datastore/mysql/mysql.go | 7 + server/datastore/mysql/schema.sql | 5 +- server/datastore/mysql/software.go | 112 ++++++----- server/datastore/mysql/software_installers.go | 13 +- server/datastore/mysql/software_test.go | 68 ++++--- server/datastore/mysql/testing_utils.go | 20 ++ server/fleet/activities.go | 34 ++++ server/fleet/app.go | 5 + server/fleet/software_installer.go | 22 +++ server/service/activities.go | 2 +- server/service/hosts.go | 4 +- server/service/integration_core_test.go | 64 ++++++- server/service/integration_enterprise_test.go | 157 ---------------- server/service/integration_mdm_test.go | 135 +++++++++++++- server/service/orbit.go | 35 +++- 21 files changed, 682 insertions(+), 323 deletions(-) create mode 100644 changes/18772-add-software-installs-to-host-activities diff --git a/changes/18772-add-software-installs-to-host-activities b/changes/18772-add-software-installs-to-host-activities new file mode 100644 index 0000000000..c0b36638fa --- /dev/null +++ b/changes/18772-add-software-installs-to-host-activities @@ -0,0 +1 @@ +* Added software installation to the host's upcoming and past activities. diff --git a/docs/Using Fleet/Audit-logs.md b/docs/Using Fleet/Audit-logs.md index d0bda3d634..eb3338d727 100644 --- a/docs/Using Fleet/Audit-logs.md +++ b/docs/Using Fleet/Audit-logs.md @@ -1127,6 +1127,29 @@ This activity contains the following fields: } ``` +## installed_software + +Generated when a software is installed on a host. + +This activity contains the following fields: +- "host_id": ID of the host. +- "host_display_name": Display name of the host. +- "install_uuid": ID of the software installation. +- "software_title": Name of the software. +- "status": Status of the software installation. + +#### Example + +```json +{ + "host_id": 1, + "host_display_name": "Anna's MacBook Pro", + "software_title": "Falcon.app", + "install_uuid": "d6cffa75-b5b5-41ef-9230-15073c8a88cf", + "status": "pending" +} +``` + diff --git a/docs/Using Fleet/Understanding-host-vitals.md b/docs/Using Fleet/Understanding-host-vitals.md index 2fe0abb033..a265bd01fe 100644 --- a/docs/Using Fleet/Understanding-host-vitals.md +++ b/docs/Using Fleet/Understanding-host-vitals.md @@ -45,7 +45,14 @@ SELECT de.encrypted, m.path FROM disk_encryption de JOIN mounts m ON m.device_al - Query: ```sql -SELECT 1 FROM bitlocker_info WHERE drive_letter = 'C:' AND protection_status = 1; +WITH encrypted(enabled) AS ( + SELECT CASE WHEN + NOT EXISTS(SELECT 1 FROM windows_optional_features WHERE name = 'BitLocker') + OR + (SELECT 1 FROM windows_optional_features WHERE name = 'BitLocker' AND state = 1) + THEN (SELECT 1 FROM bitlocker_info WHERE drive_letter = 'C:' AND protection_status = 1) + END) + SELECT 1 FROM encrypted WHERE enabled IS NOT NULL ``` ## disk_space_unix @@ -304,7 +311,7 @@ LIMIT 1; ## orbit_info -- Platforms: all +- Platforms: linux, ubuntu, debian, rhel, centos, sles, kali, gentoo, amzn, pop, arch, linuxmint, void, nixos, endeavouros, manjaro, opensuse-leap, opensuse-tumbleweed, darwin, windows - Discovery query: ```sql @@ -313,7 +320,7 @@ SELECT 1 FROM osquery_registry WHERE active = true AND registry = 'table' AND na - Query: ```sql -SELECT version FROM orbit_info +SELECT * FROM orbit_info ``` ## os_chrome @@ -779,7 +786,7 @@ select * from uptime limit 1 ## users -- Platforms: linux, darwin, windows +- Platforms: linux, ubuntu, debian, rhel, centos, sles, kali, gentoo, amzn, pop, arch, linuxmint, void, nixos, endeavouros, manjaro, opensuse-leap, opensuse-tumbleweed, darwin, windows - Query: ```sql diff --git a/server/datastore/mysql/activities.go b/server/datastore/mysql/activities.go index 3e6199dfaf..83bf7790b7 100644 --- a/server/datastore/mysql/activities.go +++ b/server/datastore/mysql/activities.go @@ -185,10 +185,25 @@ func (ds *Datastore) MarkActivitiesAsStreamed(ctx context.Context, activityIDs [ return nil } +// ListHostUpcomingActivities returns the list of activities pending execution +// or processing for the specific host. It is the "unified queue" of work to be +// done on the host. That queue is "virtual" in the sense that it pulls from a +// number of distinct tables that are task-specific (such as scripts to run, +// software to install, etc.) and provides a unified view of those upcoming +// tasks. func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint, opt fleet.ListOptions) ([]*fleet.Activity, *fleet.PaginationMetadata, error) { - const countStmt = `SELECT COUNT(*) FROM host_script_results WHERE host_id = ? AND exit_code IS NULL` + countStmts := []string{ + `SELECT COUNT(*) c FROM host_script_results WHERE host_id = :host_id AND exit_code IS NULL`, + `SELECT COUNT(*) c FROM host_software_installs WHERE host_id = :host_id AND pre_install_query_output IS NULL AND install_script_exit_code IS NULL`, + } + var count uint - if err := sqlx.GetContext(ctx, ds.reader(ctx), &count, countStmt, hostID); err != nil { + countStmt := `SELECT SUM(c) FROM ( ` + strings.Join(countStmts, " UNION ALL ") + ` ) AS counts` + countStmt, args, err := sqlx.Named(countStmt, map[string]any{"host_id": hostID}) + if err != nil { + return nil, nil, ctxerr.Wrap(ctx, err, "build count query from named args") + } + if err := sqlx.GetContext(ctx, ds.reader(ctx), &count, countStmt, args...); err != nil { return nil, nil, ctxerr.Wrap(ctx, err, "count upcoming activities") } if count == 0 { @@ -196,14 +211,15 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint } // NOTE: Be sure to update both the count and list statements if the list query is modified - const listStmt = ` - SELECT + listStmts := []string{ + // list pending scripts + `SELECT hsr.execution_id as uuid, u.name as name, u.id as user_id, u.gravatar_url as gravatar_url, u.email as user_email, - ? as activity_type, + :ran_script_type as activity_type, hsr.created_at as created_at, JSON_OBJECT( 'host_id', hsr.host_id, @@ -221,16 +237,70 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint LEFT OUTER JOIN scripts scr ON scr.id = hsr.script_id WHERE - hsr.host_id = ? AND - hsr.exit_code IS NULL - AND ( - hsr.sync_request = 0 - OR hsr.created_at >= DATE_SUB(NOW(), INTERVAL ? SECOND) - ) -` + hsr.host_id = :host_id AND + hsr.exit_code IS NULL AND + ( + hsr.sync_request = 0 OR + hsr.created_at >= DATE_SUB(NOW(), INTERVAL :max_wait_time SECOND) + ) +`, + // list pending software installs + fmt.Sprintf(`SELECT + hsi.execution_id as uuid, + u.name as name, + u.id as user_id, + u.gravatar_url as gravatar_url, + u.email as user_email, + :installed_software_type as activity_type, + hsi.created_at as created_at, + JSON_OBJECT( + 'host_id', hsi.host_id, + 'host_display_name', COALESCE(hdn.display_name, ''), + 'software_title', COALESCE(st.name, ''), + 'install_uuid', hsi.execution_id, + 'status', %s + ) as details + FROM + host_software_installs hsi + INNER JOIN + software_installers si ON si.id = hsi.software_installer_id + LEFT OUTER JOIN + software_titles st ON st.id = si.title_id + LEFT OUTER JOIN + users u ON u.id = hsi.user_id + LEFT OUTER JOIN + host_display_names hdn ON hdn.host_id = hsi.host_id + WHERE + hsi.host_id = :host_id AND + hsi.pre_install_query_output IS NULL AND + hsi.install_script_exit_code IS NULL + `, softwareInstallerHostStatusNamedQuery("")), + } seconds := int(scripts.MaxServerWaitTime.Seconds()) - args := []any{fleet.ActivityTypeRanScript{}.ActivityName(), hostID, seconds} + listStmt := ` + SELECT + uuid, + name, + user_id, + gravatar_url, + user_email, + activity_type, + created_at, + details + FROM ( ` + strings.Join(listStmts, " UNION ALL ") + ` ) AS upcoming ` + listStmt, args, err = sqlx.Named(listStmt, map[string]any{ + "host_id": hostID, + "ran_script_type": fleet.ActivityTypeRanScript{}.ActivityName(), + "installed_software_type": fleet.ActivityTypeInstalledSoftware{}.ActivityName(), + "max_wait_time": seconds, + "software_status_failed": fleet.SoftwareInstallerFailed, + "software_status_installed": fleet.SoftwareInstallerInstalled, + "software_status_pending": fleet.SoftwareInstallerPending, + }) + if err != nil { + return nil, nil, ctxerr.Wrap(ctx, err, "build list query from named args") + } stmt, args := appendListOptionsWithCursorToSQL(listStmt, args, &opt) var activities []*fleet.Activity @@ -255,7 +325,7 @@ func (ds *Datastore) ListHostPastActivities(ctx context.Context, hostID uint, op a.user_email as user_email, a.user_name as name, a.activity_type as activity_type, - a.details as details, + a.details as details, u.gravatar_url as gravatar_url, a.created_at as created_at, u.id as user_id diff --git a/server/datastore/mysql/activities_test.go b/server/datastore/mysql/activities_test.go index ec4e2b8cc9..40f2cd061a 100644 --- a/server/datastore/mysql/activities_test.go +++ b/server/datastore/mysql/activities_test.go @@ -10,9 +10,11 @@ import ( "testing" "time" + "github.com/fleetdm/fleet/v4/server/contexts/viewer" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/ptr" "github.com/fleetdm/fleet/v4/server/test" + "github.com/google/uuid" "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -305,9 +307,11 @@ func testActivityPaginationMetadata(t *testing.T, ds *Datastore) { } func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { - ctx := context.Background() + noUserCtx := context.Background() u := test.NewUser(t, ds, "user1", "user1@example.com", false) + u2 := test.NewUser(t, ds, "user2", "user2@example.com", false) + ctx := viewer.NewContext(noUserCtx, viewer.Viewer{User: u2}) // create three hosts h1 := test.NewHost(t, ds, "h1.local", "10.10.10.1", "1", "1", time.Now()) @@ -326,6 +330,41 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { }) require.NoError(t, err) + // create a couple of software installers + installer := strings.NewReader("echo") + sw1, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "install foo", + InstallerFile: installer, + StorageID: uuid.NewString(), + Filename: "foo.pkg", + Title: "foo", + Source: "apps", + Version: "0.0.1", + }) + require.NoError(t, err) + sw2, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "install bar", + InstallerFile: installer, + StorageID: uuid.NewString(), + Filename: "bar.pkg", + Title: "bar", + Source: "apps", + Version: "0.0.2", + }) + require.NoError(t, err) + sw1Meta, err := ds.GetSoftwareInstallerMetadata(ctx, sw1) + require.NoError(t, err) + sw2Meta, err := ds.GetSoftwareInstallerMetadata(ctx, sw2) + require.NoError(t, err) + + latestSoftwareInstallerUUID := func() string { + var id string + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(ctx, q, &id, `SELECT execution_id FROM host_software_installs ORDER BY id DESC LIMIT 1`) + }) + return id + } + // create some script requests for h1 hsr, err := ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h1.ID, ScriptID: &scr1.ID, ScriptContents: scr1.ScriptContents, UserID: &u.ID}) require.NoError(t, err) @@ -342,6 +381,32 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h1.ID, ScriptContents: "E"}) require.NoError(t, err) h1E := hsr.ExecutionID + // create some software installs requests for h1, make some complete + err = ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1Meta.TitleID, nil) + require.NoError(t, err) + h1FooFailed := latestSoftwareInstallerUUID() + err = ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw2Meta.TitleID, nil) + require.NoError(t, err) + h1Bar := latestSoftwareInstallerUUID() + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: h1.ID, + InstallUUID: h1FooFailed, + PreInstallConditionOutput: ptr.String(""), // pre-install failed + }) + require.NoError(t, err) + err = ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1Meta.TitleID, nil) + require.NoError(t, err) + h1FooInstalled := latestSoftwareInstallerUUID() + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: h1.ID, + InstallUUID: h1FooInstalled, + PreInstallConditionOutput: ptr.String("ok"), + InstallScriptExitCode: ptr.Int(0), + }) + require.NoError(t, err) + err = ds.InsertSoftwareInstallRequest(noUserCtx, h1.ID, sw1Meta.TitleID, nil) // no user for this one + require.NoError(t, err) + h1Foo := latestSoftwareInstallerUUID() // create a single pending request for h2, as well as a non-pending one hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h2.ID, ScriptID: &scr1.ID, ScriptContents: scr1.ScriptContents, UserID: &u.ID}) @@ -352,23 +417,43 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { _, err = ds.SetHostScriptExecutionResult(ctx, &fleet.HostScriptResultPayload{HostID: h2.ID, ExecutionID: hsr.ExecutionID, Output: "ok", ExitCode: 0}) require.NoError(t, err) h2F := hsr.ExecutionID + // add a pending software install request for h2 + err = ds.InsertSoftwareInstallRequest(ctx, h2.ID, sw2Meta.TitleID, nil) + require.NoError(t, err) + h2Bar := latestSoftwareInstallerUUID() - // no script request for h3 + // nothing for h3 + + // force-set the order of the created_at timestamps + endTime := SetOrderedCreatedAtTimestamps(t, ds, time.Now(), "host_script_results", "execution_id", h1A, h1B) + endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_software_installs", "execution_id", h1FooFailed, h1Bar) + endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_script_results", "execution_id", h1C, h1D, h1E) + endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_software_installs", "execution_id", h1FooInstalled, h1Foo) + endTime = SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_software_installs", "execution_id", h2Bar) + SetOrderedCreatedAtTimestamps(t, ds, endTime, "host_script_results", "execution_id", h2A, h2F) execIDsWithUser := map[string]bool{ - h1A: true, - h1B: true, - h1C: true, - h1D: false, - h1E: false, - h2A: true, - h2F: true, + h1A: true, + h1B: true, + h1C: true, + h1D: false, + h1E: false, + h2A: true, + h2F: true, + h1Foo: false, + h1Bar: true, + h2Bar: true, } execIDsScriptName := map[string]string{ h1A: scr1.Name, h1B: scr2.Name, h2A: scr1.Name, } + execIDsSoftwareTitle := map[string]string{ + h1Foo: "foo", + h1Bar: "bar", + h2Bar: "bar", + } cases := []struct { opts fleet.ListOptions @@ -380,43 +465,49 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { opts: fleet.ListOptions{PerPage: 2}, hostID: h1.ID, wantExecs: []string{h1A, h1B}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false, TotalResults: 5}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false, TotalResults: 7}, }, { opts: fleet.ListOptions{Page: 1, PerPage: 2}, hostID: h1.ID, - wantExecs: []string{h1C, h1D}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: true, TotalResults: 5}, + wantExecs: []string{h1Bar, h1C}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: true, TotalResults: 7}, }, { opts: fleet.ListOptions{Page: 2, PerPage: 2}, hostID: h1.ID, - wantExecs: []string{h1E}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 5}, - }, - { - opts: fleet.ListOptions{PerPage: 3}, - hostID: h1.ID, - wantExecs: []string{h1A, h1B, h1C}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false, TotalResults: 5}, - }, - { - opts: fleet.ListOptions{Page: 1, PerPage: 3}, - hostID: h1.ID, wantExecs: []string{h1D, h1E}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 5}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: true, TotalResults: 7}, }, { - opts: fleet.ListOptions{Page: 2, PerPage: 3}, + opts: fleet.ListOptions{Page: 3, PerPage: 2}, + hostID: h1.ID, + wantExecs: []string{h1Foo}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 7}, + }, + { + opts: fleet.ListOptions{PerPage: 4}, + hostID: h1.ID, + wantExecs: []string{h1A, h1B, h1Bar, h1C}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false, TotalResults: 7}, + }, + { + opts: fleet.ListOptions{Page: 1, PerPage: 4}, + hostID: h1.ID, + wantExecs: []string{h1D, h1E, h1Foo}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 7}, + }, + { + opts: fleet.ListOptions{Page: 2, PerPage: 4}, hostID: h1.ID, wantExecs: []string{}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 5}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 7}, }, { opts: fleet.ListOptions{PerPage: 3}, hostID: h2.ID, - wantExecs: []string{h2A}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: false, TotalResults: 1}, + wantExecs: []string{h2Bar, h2A}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: false, TotalResults: 2}, }, { opts: fleet.ListOptions{}, @@ -445,21 +536,37 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { require.NotNil(t, a.Details, "result %d", i) require.NoError(t, json.Unmarshal([]byte(*a.Details), &details), "result %d", i) - require.Equal(t, wantExec, details["script_execution_id"], "result %d", i) require.Equal(t, c.hostID, uint(details["host_id"].(float64)), "result %d", i) - require.Equal(t, execIDsScriptName[wantExec], details["script_name"], "result %d", i) + + var wantUser *fleet.User + switch a.Type { + case fleet.ActivityTypeRanScript{}.ActivityName(): + require.Equal(t, wantExec, details["script_execution_id"], "result %d", i) + require.Equal(t, execIDsScriptName[wantExec], details["script_name"], "result %d", i) + wantUser = u + + case fleet.ActivityTypeInstalledSoftware{}.ActivityName(): + require.Equal(t, wantExec, details["install_uuid"], "result %d", i) + require.Equal(t, execIDsSoftwareTitle[wantExec], details["software_title"], "result %d", i) + wantUser = u2 + + default: + t.Fatalf("unknown activity type %s", a.Type) + } + if execIDsWithUser[wantExec] { require.NotNil(t, a.ActorID, "result %d", i) - require.Equal(t, u.ID, *a.ActorID, "result %d", i) + require.Equal(t, wantUser.ID, *a.ActorID, "result %d", i) require.NotNil(t, a.ActorFullName, "result %d", i) - require.Equal(t, u.Name, *a.ActorFullName, "result %d", i) + require.Equal(t, wantUser.Name, *a.ActorFullName, "result %d", i) require.NotNil(t, a.ActorEmail, "result %d", i) - require.Equal(t, u.Email, *a.ActorEmail, "result %d", i) + require.Equal(t, wantUser.Email, *a.ActorEmail, "result %d", i) } else { require.Nil(t, a.ActorID, "result %d", i) require.Nil(t, a.ActorFullName, "result %d", i) require.Nil(t, a.ActorEmail, "result %d", i) } + } }) } diff --git a/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go b/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go index 6c09e213b0..478aa8b0f0 100644 --- a/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go +++ b/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS software_installers ( global_or_team_id INT(10) UNSIGNED NOT NULL DEFAULT 0, -- FK to the "software title" this installer matches - title_id int(10) unsigned DEFAULT NULL, + title_id int(10) unsigned DEFAULT NULL, -- Filename of the uploaded installer filename varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, @@ -110,6 +110,9 @@ CREATE TABLE IF NOT EXISTS host_software_installs ( -- Exit code of the post-script run after the software is installed post_install_script_exit_code int(10) DEFAULT NULL, + -- User that requested the installation, for upcoming activities + user_id int(10) unsigned DEFAULT NULL, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, @@ -120,6 +123,11 @@ CREATE TABLE IF NOT EXISTS host_software_installs ( REFERENCES software_installers (id) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT fk_host_software_installs_user_id + FOREIGN KEY (user_id) + REFERENCES users (id) + ON DELETE SET NULL, + KEY idx_host_software_installs_host_installer (host_id, software_installer_id), -- this index can be used to lookup results for a specific diff --git a/server/datastore/mysql/mysql.go b/server/datastore/mysql/mysql.go index 39ca1f7082..2b5238f1d8 100644 --- a/server/datastore/mysql/mysql.go +++ b/server/datastore/mysql/mysql.go @@ -792,6 +792,13 @@ func appendListOptionsWithCursorToSQL(sql string, params []interface{}, opts *fl } sql = fmt.Sprintf("%s ORDER BY %s %s", sql, orderKey, direction) + if opts.TestSecondaryOrderKey != "" { + direction := "ASC" + if opts.TestSecondaryOrderDirection == fleet.OrderDescending { + direction = "DESC" + } + sql += fmt.Sprintf(`, %s %s`, sanitizeColumn(opts.TestSecondaryOrderKey), direction) + } } // REVIEW: If caller doesn't supply a limit apply a default limit to insure // that an unbounded query with many results doesn't consume too much memory diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 9366e8d653..be08aec070 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -496,13 +496,16 @@ CREATE TABLE `host_software_installs` ( `install_script_exit_code` int(10) DEFAULT NULL, `post_install_script_output` text COLLATE utf8mb4_unicode_ci, `post_install_script_exit_code` int(10) DEFAULT NULL, + `user_id` int(10) unsigned DEFAULT NULL, `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `idx_host_software_installs_execution_id` (`execution_id`), KEY `fk_host_software_installs_installer_id` (`software_installer_id`), + KEY `fk_host_software_installs_user_id` (`user_id`), KEY `idx_host_software_installs_host_installer` (`host_id`,`software_installer_id`), - CONSTRAINT `fk_host_software_installs_installer_id` FOREIGN KEY (`software_installer_id`) REFERENCES `software_installers` (`id`) ON DELETE CASCADE ON UPDATE CASCADE + CONSTRAINT `fk_host_software_installs_installer_id` FOREIGN KEY (`software_installer_id`) REFERENCES `software_installers` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `fk_host_software_installs_user_id` FOREIGN KEY (`user_id`) REFERENCES `users` (`id`) ON DELETE SET NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 0e6a02b899..d2fbb3e670 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -1721,12 +1721,41 @@ func (ds *Datastore) ListCVEs(ctx context.Context, maxAge time.Duration) ([]flee return result, nil } +// colAlias is the name to be assigned to the computed status column, pass +// empty to have the value only, no column alias set. +func softwareInstallerHostStatusNamedQuery(colAlias string) string { + if colAlias != "" { + colAlias = " AS " + colAlias + } + return fmt.Sprintf(` + CASE + WHEN hsi.post_install_script_exit_code IS NOT NULL AND + hsi.post_install_script_exit_code = 0 THEN :software_status_installed + + WHEN hsi.post_install_script_exit_code IS NOT NULL AND + hsi.post_install_script_exit_code != 0 THEN :software_status_failed + + WHEN hsi.install_script_exit_code IS NOT NULL AND + hsi.install_script_exit_code = 0 THEN :software_status_installed + + WHEN hsi.install_script_exit_code IS NOT NULL AND + hsi.install_script_exit_code != 0 THEN :software_status_failed + + WHEN hsi.pre_install_query_output IS NOT NULL AND + hsi.pre_install_query_output = '' THEN :software_status_failed + + WHEN hsi.host_id IS NOT NULL THEN :software_status_pending + + ELSE NULL -- not installed from Fleet installer + END %s `, colAlias) +} + func (ds *Datastore) ListHostSoftware(ctx context.Context, hostID uint, includeAvailableForInstall bool, opts fleet.ListOptions) ([]*fleet.HostSoftwareWithInstaller, *fleet.PaginationMetadata, error) { // `status` computed column assumes that all results (pre, install and post) // are stored at once, so that if there is an exit code for the install // script and none for the post-install, it is because there is no // post-install. - const stmtInstalled = ` + stmtInstalled := fmt.Sprintf(` SELECT st.id, st.name, @@ -1734,33 +1763,14 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, hostID uint, includeA si.filename as package_available_for_install, hsi.created_at as last_install_installed_at, hsi.execution_id as last_install_install_uuid, - CASE - WHEN hsi.post_install_script_exit_code IS NOT NULL AND - hsi.post_install_script_exit_code = 0 THEN ? -- installed - - WHEN hsi.post_install_script_exit_code IS NOT NULL AND - hsi.post_install_script_exit_code != 0 THEN ? -- failed - - WHEN hsi.install_script_exit_code IS NOT NULL AND - hsi.install_script_exit_code = 0 THEN ? -- installed - - WHEN hsi.install_script_exit_code IS NOT NULL AND - hsi.install_script_exit_code != 0 THEN ? -- failed - - WHEN hsi.pre_install_query_output IS NOT NULL AND - hsi.pre_install_query_output = '' THEN ? -- failed - - WHEN hsi.host_id IS NOT NULL THEN ? -- pending - - ELSE NULL -- not installed from Fleet installer - END AS status, + %s, si.id AS installer_id -- NULL if no Fleet installer FROM software_titles st LEFT OUTER JOIN software_installers si ON st.id = si.title_id LEFT OUTER JOIN - host_software_installs hsi ON si.id = hsi.software_installer_id AND hsi.host_id = ? + host_software_installs hsi ON si.id = hsi.software_installer_id AND hsi.host_id = :host_id WHERE -- use the latest install only ( hsi.id IS NULL OR hsi.id = ( @@ -1777,12 +1787,12 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, hostID uint, includeA INNER JOIN software s ON hs.software_id = s.id WHERE - hs.host_id = ? AND + hs.host_id = :host_id AND s.title_id = st.id ) OR -- or software install has been attempted on host hsi.host_id IS NOT NULL ) -` +`, softwareInstallerHostStatusNamedQuery("status")) const stmtAvailable = ` SELECT @@ -1807,7 +1817,7 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, hostID uint, includeA INNER JOIN software s ON hs.software_id = s.id WHERE - hs.host_id = ? AND + hs.host_id = :host_id AND s.title_id = st.id ) AND -- sofware install has not been attempted on host @@ -1816,10 +1826,10 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, hostID uint, includeA FROM host_software_installs hsi WHERE - hsi.host_id = ? AND + hsi.host_id = :host_id AND hsi.software_installer_id = si.id ) AND - si.global_or_team_id = (SELECT COALESCE(h.team_id, 0) FROM hosts h WHERE h.id = ?) + si.global_or_team_id = (SELECT COALESCE(h.team_id, 0) FROM hosts h WHERE h.id = :host_id) ` const selectColNames = ` @@ -1830,48 +1840,32 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, hostID uint, includeA package_available_for_install, last_install_installed_at, last_install_install_uuid, - status, - CASE - WHEN status = ? THEN 4 -- failed - WHEN status = ? THEN 3 -- pending - WHEN status = ? THEN 2 -- installed - WHEN installer_id IS NOT NULL THEN 1 -- installer exists, not installed - ELSE 0 -- no installer exists - END AS status_sort + status ` - args := []any{ - // for status_sort - fleet.SoftwareInstallerFailed, - fleet.SoftwareInstallerPending, - fleet.SoftwareInstallerInstalled, - - // for status - fleet.SoftwareInstallerInstalled, - fleet.SoftwareInstallerFailed, - fleet.SoftwareInstallerInstalled, - fleet.SoftwareInstallerFailed, - fleet.SoftwareInstallerFailed, - fleet.SoftwareInstallerPending, - - hostID, - hostID, - } stmt := stmtInstalled if includeAvailableForInstall { stmt += ` UNION ` + stmtAvailable - args = append(args, hostID, hostID, hostID) } + stmt = selectColNames + ` FROM ( ` + stmt + ` ) AS tbl ` + // must resolve the named bindings here, before adding the searchLike which + // uses standard placeholders. + stmt, args, err := sqlx.Named(stmt, map[string]any{ + "host_id": hostID, + "software_status_failed": fleet.SoftwareInstallerFailed, + "software_status_pending": fleet.SoftwareInstallerPending, + "software_status_installed": fleet.SoftwareInstallerInstalled, + }) + if err != nil { + return nil, nil, ctxerr.Wrap(ctx, err, "build named query for list host software") + } + if opts.MatchQuery != "" { stmt += " WHERE TRUE " // searchLike adds a "AND " stmt, args = searchLike(stmt, args, opts.MatchQuery, "name") } - // apply default sort (adding source just to make it deterministic) - if opts.OrderKey == "" { - stmt += ` ORDER BY status_sort DESC, name ASC, source ASC ` - } stmt, _ = appendListOptionsToSQL(stmt, &opts) type hostSoftware struct { @@ -2036,7 +2030,8 @@ func (ds *Datastore) SetHostSoftwareInstallResult(ctx context.Context, result *f post_install_script_exit_code = ?, post_install_script_output = ? WHERE - execution_id = ? + execution_id = ? AND + host_id = ? ` res, err := ds.writer(ctx).ExecContext(ctx, stmt, result.PreInstallConditionOutput, @@ -2045,6 +2040,7 @@ func (ds *Datastore) SetHostSoftwareInstallResult(ctx context.Context, result *f result.PostInstallScriptExitCode, result.PostInstallScriptOutput, result.InstallUUID, + result.HostID, ) if err != nil { return ctxerr.Wrap(ctx, err, "update host software installation result") diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 2ef91aeacc..f9d7f6a5ee 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" + "github.com/fleetdm/fleet/v4/server/authz" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/google/uuid" @@ -200,9 +201,9 @@ func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID ui const ( insertStmt = ` INSERT INTO host_software_installs - (execution_id, host_id, software_installer_id) + (execution_id, host_id, software_installer_id, user_id) VALUES - (?, ?, ?) + (?, ?, ?, ?) ` getInstallerIDStmt = `SELECT id FROM software_installers WHERE title_id = ? AND global_or_team_id = ?` @@ -231,10 +232,15 @@ func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID ui return ctxerr.Wrap(ctx, err, "inserting new install software request") } + var userID *uint + if ctxUser := authz.UserFromContext(ctx); ctxUser != nil { + userID = &ctxUser.ID + } _, err = ds.writer(ctx).ExecContext(ctx, insertStmt, uuid.NewString(), hostID, installerID, + userID, ) return ctxerr.Wrap(ctx, err, "inserting new install software request") @@ -266,7 +272,8 @@ SELECT ELSE NULL -- not installed from Fleet installer END, '') AS status, si.filename AS software_package, - h.team_id AS host_team_id + h.team_id AS host_team_id, + hsi.user_id AS user_id FROM host_software_installs hsi JOIN hosts h ON h.id = hsi.host_id diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index 882ac86a05..cf07aecacf 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -2937,7 +2937,7 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { ctx := context.Background() host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) otherHost := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now()) - opts := fleet.ListOptions{PerPage: 10, IncludeMetadata: true} + opts := fleet.ListOptions{PerPage: 10, IncludeMetadata: true, OrderKey: "name", TestSecondaryOrderKey: "source"} // no software yet sw, meta, err := ds.ListHostSoftware(ctx, host.ID, false, opts) @@ -3092,10 +3092,10 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { globalOrTeamID = tm.ID } res, err := q.ExecContext(ctx, ` - INSERT INTO software_installers - (team_id, global_or_team_id, title_id, filename, version, install_script_content_id, storage_id) - VALUES - (?, ?, ?, ?, ?, ?, unhex(?))`, + INSERT INTO software_installers + (team_id, global_or_team_id, title_id, filename, version, install_script_content_id, storage_id) + VALUES + (?, ?, ?, ?, ?, ?, unhex(?))`, teamID, globalOrTeamID, titleID, fmt.Sprintf("installer-%d.pkg", i), fmt.Sprintf("v%d.0.0", i), scriptContentID, hex.EncodeToString([]byte("test"))) if err != nil { return err @@ -3109,7 +3109,7 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { // swi1 is pending (all results are NULL) _, err = q.ExecContext(ctx, ` - INSERT INTO host_software_installs (execution_id, host_id, software_installer_id) VALUES (?, ?, ?)`, + INSERT INTO host_software_installs (execution_id, host_id, software_installer_id) VALUES (?, ?, ?)`, "uuid1", host.ID, swi1Pending) if err != nil { return err @@ -3117,8 +3117,8 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { // swi2 is installed _, err = q.ExecContext(ctx, ` - INSERT INTO host_software_installs (execution_id, host_id, software_installer_id, pre_install_query_output, install_script_exit_code, post_install_script_exit_code) - VALUES (?, ?, ?, ?, ?, ?)`, + INSERT INTO host_software_installs (execution_id, host_id, software_installer_id, pre_install_query_output, install_script_exit_code, post_install_script_exit_code) + VALUES (?, ?, ?, ?, ?, ?)`, "uuid2", host.ID, swi2Installed, "ok", 0, 0) if err != nil { return err @@ -3126,14 +3126,14 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { // swi3 is failed, also add an install request on the other host _, err = q.ExecContext(ctx, ` - INSERT INTO host_software_installs (execution_id, host_id, software_installer_id, pre_install_query_output, install_script_exit_code) - VALUES (?, ?, ?, ?, ?)`, + INSERT INTO host_software_installs (execution_id, host_id, software_installer_id, pre_install_query_output, install_script_exit_code) + VALUES (?, ?, ?, ?, ?)`, "uuid3", host.ID, swi3Failed, "ok", 1) if err != nil { return err } _, err = q.ExecContext(ctx, ` - INSERT INTO host_software_installs (execution_id, host_id, software_installer_id) VALUES (?, ?, ?)`, + INSERT INTO host_software_installs (execution_id, host_id, software_installer_id) VALUES (?, ?, ?)`, uuid.NewString(), otherHost.ID, swi3Failed) if err != nil { return err @@ -3141,7 +3141,7 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { // swi4 is available (no install request), but add a pending request on the other host _, err = q.ExecContext(ctx, ` - INSERT INTO host_software_installs (execution_id, host_id, software_installer_id) VALUES (?, ?, ?)`, + INSERT INTO host_software_installs (execution_id, host_id, software_installer_id) VALUES (?, ?, ?)`, uuid.NewString(), otherHost.ID, swi4Available) if err != nil { return err @@ -3193,22 +3193,34 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { PackageAvailableForInstall: ptr.String("installer-4.pkg"), } - // request without available software, returns failed first, pending, installed, other + // request without available software sw, meta, err = ds.ListHostSoftware(ctx, host.ID, false, opts) require.NoError(t, err) require.Equal(t, &fleet.PaginationMetadata{}, meta) compareResults([]*fleet.HostSoftwareWithInstaller{ - expected["i1"], expected["b"], expected["i0"], expected["a1"], expected["a2"], expected["c"], expected["d"], + expected["a1"], expected["a2"], expected["b"], expected["c"], expected["d"], expected["i0"], expected["i1"], }, sw) - // request with available software, returns failed first, pending, installed, available, other + // request with available software sw, meta, err = ds.ListHostSoftware(ctx, host.ID, true, opts) require.NoError(t, err) require.Equal(t, &fleet.PaginationMetadata{}, meta) compareResults([]*fleet.HostSoftwareWithInstaller{ - expected["i1"], expected["b"], expected["i0"], expected["i2"], expected["a1"], expected["a2"], expected["c"], expected["d"], + expected["a1"], expected["a2"], expected["b"], expected["c"], expected["d"], expected["i0"], expected["i1"], expected["i2"], }, sw) + // request in descending order + opts.OrderDirection = fleet.OrderDescending + opts.TestSecondaryOrderDirection = fleet.OrderDescending + sw, meta, err = ds.ListHostSoftware(ctx, host.ID, false, opts) + require.NoError(t, err) + require.Equal(t, &fleet.PaginationMetadata{}, meta) + compareResults([]*fleet.HostSoftwareWithInstaller{ + expected["i1"], expected["i0"], expected["d"], expected["c"], expected["b"], expected["a2"], expected["a1"], + }, sw) + opts.OrderDirection = fleet.OrderAscending + opts.TestSecondaryOrderDirection = fleet.OrderAscending + // record a new install request for i1, this time as pending, and mark install request for b (swi1) as failed time.Sleep(time.Second) // ensure the timestamp is later err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ @@ -3221,8 +3233,8 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { // swi3 has a new install request pending _, err = q.ExecContext(ctx, ` - INSERT INTO host_software_installs (execution_id, host_id, software_installer_id) - VALUES (?, ?, ?)`, + INSERT INTO host_software_installs (execution_id, host_id, software_installer_id) + VALUES (?, ?, ?)`, "uuid4", host.ID, swi3Failed) if err != nil { return err @@ -3248,20 +3260,20 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { PackageAvailableForInstall: ptr.String("installer-2.pkg"), } - // request without available software, returns failed first, pending, installed, other + // request without available software sw, meta, err = ds.ListHostSoftware(ctx, host.ID, false, opts) require.NoError(t, err) require.Equal(t, &fleet.PaginationMetadata{}, meta) compareResults([]*fleet.HostSoftwareWithInstaller{ - expected["b"], expected["i1"], expected["i0"], expected["a1"], expected["a2"], expected["c"], expected["d"], + expected["a1"], expected["a2"], expected["b"], expected["c"], expected["d"], expected["i0"], expected["i1"], }, sw) - // request with available software, returns failed first, pending, installed, available, other + // request with available software sw, meta, err = ds.ListHostSoftware(ctx, host.ID, true, opts) require.NoError(t, err) require.Equal(t, &fleet.PaginationMetadata{}, meta) compareResults([]*fleet.HostSoftwareWithInstaller{ - expected["b"], expected["i1"], expected["i0"], expected["i2"], expected["a1"], expected["a2"], expected["c"], expected["d"], + expected["a1"], expected["a2"], expected["b"], expected["c"], expected["d"], expected["i0"], expected["i1"], expected["i2"], }, sw) // create a new host in the team, with no software @@ -3307,19 +3319,19 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { { opts: fleet.ListOptions{PerPage: 3}, withAvailable: false, - wantNames: []string{expected["b"].Name, expected["i1"].Name, expected["i0"].Name}, + wantNames: []string{expected["a1"].Name, expected["a2"].Name, expected["b"].Name}, wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false}, }, { opts: fleet.ListOptions{Page: 1, PerPage: 3}, withAvailable: false, - wantNames: []string{expected["a1"].Name, expected["a2"].Name, expected["c"].Name}, + wantNames: []string{expected["c"].Name, expected["d"].Name, expected["i0"].Name}, wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: true}, }, { opts: fleet.ListOptions{Page: 2, PerPage: 3}, withAvailable: false, - wantNames: []string{expected["d"].Name}, + wantNames: []string{expected["i1"].Name}, wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true}, }, { @@ -3331,13 +3343,13 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { { opts: fleet.ListOptions{PerPage: 4}, withAvailable: true, - wantNames: []string{expected["b"].Name, expected["i1"].Name, expected["i0"].Name, expected["i2"].Name}, + wantNames: []string{expected["a1"].Name, expected["a2"].Name, expected["b"].Name, expected["c"].Name}, wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false}, }, { opts: fleet.ListOptions{Page: 1, PerPage: 4}, withAvailable: true, - wantNames: []string{expected["a1"].Name, expected["a2"].Name, expected["c"].Name, expected["d"].Name}, + wantNames: []string{expected["d"].Name, expected["i0"].Name, expected["i1"].Name, expected["i2"].Name}, wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true}, }, { @@ -3351,6 +3363,8 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { t.Run(fmt.Sprintf("%t: %#v", c.withAvailable, c.opts), func(t *testing.T) { // always include metadata c.opts.IncludeMetadata = true + c.opts.OrderKey = "name" + c.opts.TestSecondaryOrderKey = "source" sw, meta, err := ds.ListHostSoftware(ctx, host.ID, c.withAvailable, c.opts) require.NoError(t, err) diff --git a/server/datastore/mysql/testing_utils.go b/server/datastore/mysql/testing_utils.go index 4671b63b77..b46e07028a 100644 --- a/server/datastore/mysql/testing_utils.go +++ b/server/datastore/mysql/testing_utils.go @@ -467,3 +467,23 @@ func GetAggregatedStats(ctx context.Context, ds *Datastore, aggregate fleet.Aggr err := sqlx.GetContext(ctx, ds.reader(ctx), &result, stmt, id, aggregate) return result, err } + +// SetOrderedCreatedAtTimestamps enforces an ordered sequence of created_at +// timestamps in a database table. This can be useful in tests instead of +// adding time.Sleep calls to just force specific ordered timestamps for the +// test entries of interest, and it doesn't slow down the unit test. +// +// The first timestamp will be after afterTime, and each provided key will have +// a timestamp incremented by 1s. +func SetOrderedCreatedAtTimestamps(t testing.TB, ds *Datastore, afterTime time.Time, table, keyCol string, keys ...any) time.Time { + now := afterTime + for i := 0; i < len(keys); i++ { + now = afterTime.Add(time.Second) + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(context.Background(), + fmt.Sprintf(`UPDATE %s SET created_at=? WHERE %s=?`, table, keyCol), now, keys[i]) + return err + }) + } + return now +} diff --git a/server/fleet/activities.go b/server/fleet/activities.go index 00a2964455..1eccc63c94 100644 --- a/server/fleet/activities.go +++ b/server/fleet/activities.go @@ -90,6 +90,8 @@ var ActivityDetailsList = []ActivityDetails{ ActivityTypeEditedDeclarationProfile{}, ActivityTypeResentConfigurationProfile{}, + + ActivityTypeInstalledSoftware{}, } type ActivityDetails interface { @@ -1414,6 +1416,38 @@ func (a ActivityTypeResentConfigurationProfile) Documentation() (activity string }` } +type ActivityTypeInstalledSoftware struct { + HostID uint `json:"host_id"` + HostDisplayName string `json:"host_display_name"` + SoftwareTitle string `json:"software_title"` + InstallUUID string `json:"install_uuid"` + Status string `json:"status"` +} + +func (a ActivityTypeInstalledSoftware) ActivityName() string { + return "installed_software" +} + +func (a ActivityTypeInstalledSoftware) HostIDs() []uint { + return []uint{a.HostID} +} + +func (a ActivityTypeInstalledSoftware) Documentation() (activity, details, detailsExample string) { + return `Generated when a software is installed on a host.`, + `This activity contains the following fields: +- "host_id": ID of the host. +- "host_display_name": Display name of the host. +- "install_uuid": ID of the software installation. +- "software_title": Name of the software. +- "status": Status of the software installation.`, `{ + "host_id": 1, + "host_display_name": "Anna's MacBook Pro", + "software_title": "Falcon.app", + "install_uuid": "d6cffa75-b5b5-41ef-9230-15073c8a88cf", + "status": "pending" +}` +} + // LogRoleChangeActivities logs activities for each role change, globally and one for each change in teams. func LogRoleChangeActivities(ctx context.Context, ds Datastore, adminUser *User, oldGlobalRole *string, oldTeamRoles []UserTeam, user *User) error { if user.GlobalRole != nil && (oldGlobalRole == nil || *oldGlobalRole != *user.GlobalRole) { diff --git a/server/fleet/app.go b/server/fleet/app.go index 8f4525a595..957a20befb 100644 --- a/server/fleet/app.go +++ b/server/fleet/app.go @@ -997,6 +997,11 @@ type ListOptions struct { After string `query:"after,optional"` // Used to request the metadata of a query IncludeMetadata bool + + // The following fields are for tests, to ensure a deterministic sort order + // when the single-column order key is not unique. + TestSecondaryOrderKey string `query:"-,optional"` + TestSecondaryOrderDirection OrderDirection `query:"-,optional"` } func (l ListOptions) Empty() bool { diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index de0f927989..f4f188610d 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -135,6 +135,8 @@ type HostSoftwareInstallerResult struct { // HostTeamID is the team ID of the host on which this software install was attempted. This // field is not sent in the response, it is only used for internal authorization. HostTeamID *uint `json:"-" db:"host_team_id"` + // UserID is the user ID that requested the software installation on that host. + UserID *uint `json:"-" db:"user_id"` } type HostSoftwareInstallerResultAuthz struct { @@ -229,3 +231,23 @@ type HostSoftwareInstallResultPayload struct { PostInstallScriptExitCode *int `json:"post_install_script_exit_code"` PostInstallScriptOutput *string `json:"post_install_script_output"` } + +// Status returns the status computed from the result payload. It should match the logic +// found in the database-computed status (see +// softwareInstallerHostStatusNamedQuery in mysql/software.go). +func (h *HostSoftwareInstallResultPayload) Status() SoftwareInstallerStatus { + switch { + case h.PostInstallScriptExitCode != nil && *h.PostInstallScriptExitCode == 0: + return SoftwareInstallerInstalled + case h.PostInstallScriptExitCode != nil && *h.PostInstallScriptExitCode != 0: + return SoftwareInstallerFailed + case h.InstallScriptExitCode != nil && *h.InstallScriptExitCode == 0: + return SoftwareInstallerInstalled + case h.InstallScriptExitCode != nil && *h.InstallScriptExitCode != 0: + return SoftwareInstallerFailed + case h.PreInstallConditionOutput != nil && *h.PreInstallConditionOutput == "": + return SoftwareInstallerFailed + default: + return SoftwareInstallerPending + } +} diff --git a/server/service/activities.go b/server/service/activities.go index a2976b3e14..b45e993625 100644 --- a/server/service/activities.go +++ b/server/service/activities.go @@ -141,7 +141,7 @@ func (svc *Service) ListHostPastActivities(ctx context.Context, hostID uint, opt // cursor-based pagination is not supported for past activities opt.After = "" - // custom ordering is not supported, always by date (oldest first) + // custom ordering is not supported, always by date (newest first) opt.OrderKey = "created_at" opt.OrderDirection = fleet.OrderDescending // no matching query support diff --git a/server/service/hosts.go b/server/service/hosts.go index 23d2e96247..92a8e35f9c 100644 --- a/server/service/hosts.go +++ b/server/service/hosts.go @@ -2522,8 +2522,8 @@ func (svc *Service) ListHostSoftware(ctx context.Context, hostID uint, opts flee // cursor-based pagination is not supported opts.After = "" - // custom ordering is not supported - opts.OrderKey = "" + // custom ordering is not supported, always by name (but asc/desc is configurable) + opts.OrderKey = "name" // always include metadata opts.IncludeMetadata = true diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index 35576136ed..1e9c7b1a43 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -11090,6 +11090,14 @@ func (s *integrationTestSuite) TestListHostUpcomingActivities() { // verifying that the service layer passes the proper options and the // rendering of the response. + latestSoftwareInstallerUUID := func() string { + var id string + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(ctx, q, &id, `SELECT execution_id FROM host_software_installs ORDER BY id DESC LIMIT 1`) + }) + return id + } + host1, err := s.ds.NewHost(ctx, &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -11103,6 +11111,7 @@ func (s *integrationTestSuite) TestListHostUpcomingActivities() { }) require.NoError(t, err) + // create script execution requests hsr, err := s.ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: host1.ID, ScriptContents: "A", SyncRequest: true}) require.NoError(t, err) h1A := hsr.ExecutionID @@ -11119,7 +11128,29 @@ func (s *integrationTestSuite) TestListHostUpcomingActivities() { require.NoError(t, err) h1E := hsr.ExecutionID - // modify the timestamp h1D to simulate an script that has + // create a software installation request + sw1, err := s.ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "install foo", + InstallerFile: strings.NewReader("echo"), + StorageID: uuid.NewString(), + Filename: "foo.pkg", + Title: "foo", + Source: "apps", + Version: "0.0.1", + }) + require.NoError(t, err) + s1Meta, err := s.ds.GetSoftwareInstallerMetadata(ctx, sw1) + require.NoError(t, err) + err = s.ds.InsertSoftwareInstallRequest(ctx, host1.ID, s1Meta.TitleID, nil) + require.NoError(t, err) + h1Foo := latestSoftwareInstallerUUID() + + // force an order to the activities + endTime := mysql.SetOrderedCreatedAtTimestamps(t, s.ds, time.Now(), "host_script_results", "execution_id", h1A, h1B) + endTime = mysql.SetOrderedCreatedAtTimestamps(t, s.ds, endTime, "host_software_installs", "execution_id", h1Foo) + mysql.SetOrderedCreatedAtTimestamps(t, s.ds, endTime, "host_script_results", "execution_id", h1C, h1D, h1E) + + // modify the timestamp h1A and h1B to simulate an script that has // been pending for a long time mysql.ExecAdhocSQL(t, s.ds, func(tx sqlx.ExtContext) error { _, err := tx.ExecContext(ctx, "UPDATE host_script_results SET created_at = ? WHERE execution_id IN (?, ?)", time.Now().Add(-24*time.Hour), h1A, h1B) @@ -11132,32 +11163,37 @@ func (s *integrationTestSuite) TestListHostUpcomingActivities() { wantMeta *fleet.PaginationMetadata }{ { - wantExecs: []string{h1B, h1C, h1D, h1E}, + wantExecs: []string{h1B, h1Foo, h1C, h1D, h1E}, wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: false}, }, { queries: []string{"per_page", "2"}, - wantExecs: []string{h1B, h1C}, + wantExecs: []string{h1B, h1Foo}, wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false}, }, { queries: []string{"per_page", "2", "page", "1"}, - wantExecs: []string{h1D, h1E}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true}, + wantExecs: []string{h1C, h1D}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: true}, }, { queries: []string{"per_page", "2", "page", "2"}, + wantExecs: []string{h1E}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true}, + }, + { + queries: []string{"per_page", "2", "page", "3"}, wantExecs: nil, wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true}, }, { queries: []string{"per_page", "3"}, - wantExecs: []string{h1B, h1C, h1D}, + wantExecs: []string{h1B, h1Foo, h1C}, wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false}, }, { queries: []string{"per_page", "3", "page", "1"}, - wantExecs: []string{h1E}, + wantExecs: []string{h1D, h1E}, wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true}, }, { @@ -11172,7 +11208,7 @@ func (s *integrationTestSuite) TestListHostUpcomingActivities() { queryArgs := c.queries s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/activities/upcoming", host1.ID), nil, http.StatusOK, &listResp, queryArgs...) - require.Equal(t, uint(5), listResp.Count) + require.Equal(t, uint(6), listResp.Count) require.Equal(t, len(c.wantExecs), len(listResp.Activities)) require.Equal(t, c.wantMeta, listResp.Meta) @@ -11182,12 +11218,20 @@ func (s *integrationTestSuite) TestListHostUpcomingActivities() { for i, a := range listResp.Activities { require.Zero(t, a.ID) require.NotEmpty(t, a.UUID) - require.Equal(t, fleet.ActivityTypeRanScript{}.ActivityName(), a.Type) + require.Contains(t, []string{ + fleet.ActivityTypeRanScript{}.ActivityName(), + fleet.ActivityTypeInstalledSoftware{}.ActivityName(), + }, a.Type) var details map[string]any require.NotNil(t, a.Details) require.NoError(t, json.Unmarshal(*a.Details, &details)) - gotExecs[i] = details["script_execution_id"].(string) + switch a.Type { + case fleet.ActivityTypeRanScript{}.ActivityName(): + gotExecs[i] = details["script_execution_id"].(string) + case fleet.ActivityTypeInstalledSoftware{}.ActivityName(): + gotExecs[i] = details["install_uuid"].(string) + } } } require.Equal(t, c.wantExecs, gotExecs) diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index fa82983d66..90c875296a 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -5,7 +5,6 @@ import ( "context" "database/sql" "encoding/base64" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -8652,162 +8651,6 @@ func (s *integrationEnterpriseTestSuite) TestListHostSoftware() { // TODO(mna): more advanced integration tests with Software Installers once the APIs are in place. } -func (s *integrationEnterpriseTestSuite) TestHostSoftwareInstallResult() { - ctx := context.Background() - t := s.T() - - host := createOrbitEnrolledHost(t, "linux", "", s.ds) - - // create a software installer and some host install requests - // TODO(mna): replace with API calls once they are available - mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { - installScript := `echo 'foo'` - res, err := q.ExecContext(ctx, `INSERT INTO script_contents (md5_checksum, contents) VALUES (UNHEX(md5(?)), ?)`, installScript, installScript) - if err != nil { - return err - } - scriptContentID, _ := res.LastInsertId() - - res, err = q.ExecContext(ctx, `INSERT INTO software_titles (name, source) VALUES ('foo', 'apps')`) - if err != nil { - return err - } - titleID, _ := res.LastInsertId() - - res, err = q.ExecContext(ctx, ` - INSERT INTO software_installers - (title_id, filename, version, install_script_content_id, storage_id) - VALUES - (?, ?, ?, ?, unhex(?))`, - titleID, "installer.pkg", "v1.0.0", scriptContentID, hex.EncodeToString([]byte("test"))) - if err != nil { - return err - } - id, _ := res.LastInsertId() - - // create some install requests for the host - for i := 0; i < 3; i++ { - _, err = q.ExecContext(ctx, ` - INSERT INTO host_software_installs (execution_id, host_id, software_installer_id) VALUES (?, ?, ?)`, - fmt.Sprintf("uuid%d", i), host.ID, id) - if err != nil { - return err - } - } - return nil - }) - - // TODO(mna): replace with API calls once they are available - type result struct { - HostID uint `db:"host_id"` - InstallUUID string `db:"execution_id"` - PreInstallConditionOutput *string `db:"pre_install_query_output"` - InstallScriptExitCode *int `db:"install_script_exit_code"` - InstallScriptOutput *string `db:"install_script_output"` - PostInstallScriptExitCode *int `db:"post_install_script_exit_code"` - PostInstallScriptOutput *string `db:"post_install_script_output"` - } - checkResults := func(want result) { - var got result - mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { - return sqlx.GetContext(ctx, q, &got, - `SELECT - host_id, - execution_id, - pre_install_query_output, - install_script_exit_code, - install_script_output, - post_install_script_exit_code, - post_install_script_output - FROM - host_software_installs - WHERE execution_id = ?`, want.InstallUUID) - }) - assert.Equal(t, want.HostID, got.HostID) - assert.Equal(t, want.InstallUUID, got.InstallUUID) - if want.PreInstallConditionOutput == nil { - assert.Nil(t, got.PreInstallConditionOutput) - } else { - assert.NotNil(t, got.PreInstallConditionOutput) - assert.Equal(t, *want.PreInstallConditionOutput, *got.PreInstallConditionOutput) - } - assert.Equal(t, want.InstallScriptExitCode, got.InstallScriptExitCode) - if want.InstallScriptOutput == nil { - assert.Nil(t, got.InstallScriptOutput) - } else { - assert.NotNil(t, got.InstallScriptOutput) - assert.Equal(t, *want.InstallScriptOutput, *got.InstallScriptOutput) - } - assert.Equal(t, want.PostInstallScriptExitCode, got.PostInstallScriptExitCode) - if want.PostInstallScriptOutput == nil { - assert.Nil(t, got.PostInstallScriptOutput) - } else { - assert.NotNil(t, got.PostInstallScriptOutput) - assert.Equal(t, *want.PostInstallScriptOutput, *got.PostInstallScriptOutput) - } - } - - s.Do("POST", "/api/fleet/orbit/software_install/result", - json.RawMessage(fmt.Sprintf(`{ - "orbit_node_key": %q, - "install_uuid": "uuid0", - "pre_install_condition_output": "1", - "install_script_exit_code": 1, - "install_script_output": "failed" - }`, *host.OrbitNodeKey)), - http.StatusNoContent) - checkResults(result{ - HostID: host.ID, - InstallUUID: "uuid0", - PreInstallConditionOutput: ptr.String("1"), - InstallScriptExitCode: ptr.Int(1), - InstallScriptOutput: ptr.String("failed"), - }) - - s.Do("POST", "/api/fleet/orbit/software_install/result", - json.RawMessage(fmt.Sprintf(`{ - "orbit_node_key": %q, - "install_uuid": "uuid1", - "pre_install_condition_output": "" - }`, *host.OrbitNodeKey)), - http.StatusNoContent) - checkResults(result{ - HostID: host.ID, - InstallUUID: "uuid1", - PreInstallConditionOutput: ptr.String(""), - }) - - s.Do("POST", "/api/fleet/orbit/software_install/result", - json.RawMessage(fmt.Sprintf(`{ - "orbit_node_key": %q, - "install_uuid": "uuid2", - "pre_install_condition_output": "1", - "install_script_exit_code": 0, - "install_script_output": "success", - "post_install_script_exit_code": 1, - "post_install_script_output": "failed" - }`, *host.OrbitNodeKey)), - http.StatusNoContent) - checkResults(result{ - HostID: host.ID, - InstallUUID: "uuid2", - PreInstallConditionOutput: ptr.String("1"), - InstallScriptExitCode: ptr.Int(0), - InstallScriptOutput: ptr.String("success"), - PostInstallScriptExitCode: ptr.Int(1), - PostInstallScriptOutput: ptr.String("failed"), - }) - - // non-existing installation uuid - s.Do("POST", "/api/fleet/orbit/software_install/result", - json.RawMessage(fmt.Sprintf(`{ - "orbit_node_key": %q, - "install_uuid": "uuid-no-such", - "pre_install_condition_output": "" - }`, *host.OrbitNodeKey)), - http.StatusNotFound) -} - func genDistributedReqWithPolicyResults(host *fleet.Host, policyResults map[uint]*bool) submitDistributedQueryResultsRequestShim { var ( results = make(map[string]json.RawMessage) diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 01a594b967..1b3fcd0c1a 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -8640,14 +8640,6 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadDownloadAndDelete() func (s *integrationMDMTestSuite) TestSoftwareInstallerNewInstallRequest() { t := s.T() - getTitleID := func(t *testing.T, title string, source string) uint { - var id uint - mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { - return sqlx.GetContext(context.Background(), q, &id, `SELECT id FROM software_titles WHERE name = ? AND source = ? AND browser = ''`, title, source) - }) - return id - } - var resp installSoftwareResponse // non-existent host s.DoJSON("POST", "/api/v1/fleet/hosts/1/software/install/1", nil, http.StatusNotFound, &resp) @@ -8686,7 +8678,7 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerNewInstallRequest() { s.uploadSoftwareInstaller(payload, http.StatusOK, "") // install script request succeeds - titleID := getTitleID(t, payload.Title, "deb_packages") + titleID := getSoftwareTitleID(t, s.ds, payload.Title, "deb_packages") resp = installSoftwareResponse{} s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/hosts/%d/software/install/%d", h.ID, titleID), nil, http.StatusAccepted, &resp) @@ -8709,6 +8701,123 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerNewInstallRequest() { require.Equal(t, fleet.SoftwareInstallerPending, results.Status) } +func (s *integrationMDMTestSuite) TestHostSoftwareInstallResult() { + ctx := context.Background() + t := s.T() + + host := createOrbitEnrolledHost(t, "linux", "", s.ds) + + // create a software installer and some host install requests + payload := &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "install script", + PreInstallQuery: "pre install query", + PostInstallScript: "post install script", + Filename: "ruby.deb", + Title: "ruby", + } + s.uploadSoftwareInstaller(payload, http.StatusOK, "") + titleID := getSoftwareTitleID(t, s.ds, payload.Title, "deb_packages") + + latestInstallUUID := func() string { + var id string + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(ctx, q, &id, `SELECT execution_id FROM host_software_installs ORDER BY id DESC LIMIT 1`) + }) + return id + } + + // create some install requests for the host + installUUIDs := make([]string, 3) + for i := 0; i < len(installUUIDs); i++ { + resp := installSoftwareResponse{} + s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/hosts/%d/software/install/%d", host.ID, titleID), nil, http.StatusAccepted, &resp) + installUUIDs[i] = latestInstallUUID() + } + + type result struct { + HostID uint + InstallUUID string + Status fleet.SoftwareInstallerStatus + } + checkResults := func(want result) { + var resp getSoftwareInstallResultsResponse + s.DoJSON("GET", "/api/v1/fleet/software/install/results/"+want.InstallUUID, nil, http.StatusOK, &resp) + + assert.Equal(t, want.HostID, resp.Results.HostID) + assert.Equal(t, want.InstallUUID, resp.Results.InstallUUID) + assert.Equal(t, want.Status, resp.Results.Status) + } + + s.Do("POST", "/api/fleet/orbit/software_install/result", + json.RawMessage(fmt.Sprintf(`{ + "orbit_node_key": %q, + "install_uuid": %q, + "pre_install_condition_output": "1", + "install_script_exit_code": 1, + "install_script_output": "failed" + }`, *host.OrbitNodeKey, installUUIDs[0])), + http.StatusNoContent) + checkResults(result{ + HostID: host.ID, + InstallUUID: installUUIDs[0], + Status: fleet.SoftwareInstallerFailed, + }) + wantAct := fleet.ActivityTypeInstalledSoftware{ + HostID: host.ID, + HostDisplayName: host.DisplayName(), + SoftwareTitle: payload.Title, + InstallUUID: installUUIDs[0], + Status: string(fleet.SoftwareInstallerFailed), + } + s.lastActivityMatches(wantAct.ActivityName(), string(jsonMustMarshal(t, wantAct)), 0) + + s.Do("POST", "/api/fleet/orbit/software_install/result", + json.RawMessage(fmt.Sprintf(`{ + "orbit_node_key": %q, + "install_uuid": %q, + "pre_install_condition_output": "" + }`, *host.OrbitNodeKey, installUUIDs[1])), + http.StatusNoContent) + checkResults(result{ + HostID: host.ID, + InstallUUID: installUUIDs[1], + Status: fleet.SoftwareInstallerFailed, + }) + wantAct.InstallUUID = installUUIDs[1] + s.lastActivityOfTypeMatches(wantAct.ActivityName(), string(jsonMustMarshal(t, wantAct)), 0) + + s.Do("POST", "/api/fleet/orbit/software_install/result", + json.RawMessage(fmt.Sprintf(`{ + "orbit_node_key": %q, + "install_uuid": %q, + "pre_install_condition_output": "1", + "install_script_exit_code": 0, + "install_script_output": "success", + "post_install_script_exit_code": 0, + "post_install_script_output": "ok" + }`, *host.OrbitNodeKey, installUUIDs[2])), + http.StatusNoContent) + checkResults(result{ + HostID: host.ID, + InstallUUID: installUUIDs[2], + Status: fleet.SoftwareInstallerInstalled, + }) + wantAct.InstallUUID = installUUIDs[2] + wantAct.Status = string(fleet.SoftwareInstallerInstalled) + lastActID := s.lastActivityOfTypeMatches(wantAct.ActivityName(), string(jsonMustMarshal(t, wantAct)), 0) + + // non-existing installation uuid + s.Do("POST", "/api/fleet/orbit/software_install/result", + json.RawMessage(fmt.Sprintf(`{ + "orbit_node_key": %q, + "install_uuid": "uuid-no-such", + "pre_install_condition_output": "" + }`, *host.OrbitNodeKey)), + http.StatusNotFound) + // no new activity created + s.lastActivityOfTypeMatches(wantAct.ActivityName(), string(jsonMustMarshal(t, wantAct)), lastActID) +} + func (s *integrationMDMTestSuite) uploadSoftwareInstaller(payload *fleet.UploadSoftwareInstallerPayload, expectedStatus int, expectedError string) { t := s.T() openFile := func(name string) *os.File { @@ -8755,3 +8864,11 @@ func (s *integrationMDMTestSuite) uploadSoftwareInstaller(payload *fleet.UploadS require.Contains(t, errMsg, expectedError) } } + +func getSoftwareTitleID(t *testing.T, ds *mysql.Datastore, title, source string) uint { + var id uint + mysql.ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(context.Background(), q, &id, `SELECT id FROM software_titles WHERE name = ? AND source = ? AND browser = ''`, title, source) + }) + return id +} diff --git a/server/service/orbit.go b/server/service/orbit.go index 645cef2c03..eb6297a725 100644 --- a/server/service/orbit.go +++ b/server/service/orbit.go @@ -890,6 +890,37 @@ func (svc *Service) SaveHostSoftwareInstallResult(ctx context.Context, result *f // always use the authenticated host's ID as host_id result.HostID = host.ID - err := svc.ds.SetHostSoftwareInstallResult(ctx, result) - return ctxerr.Wrap(ctx, err, "save host software installation result") + if err := svc.ds.SetHostSoftwareInstallResult(ctx, result); err != nil { + return ctxerr.Wrap(ctx, err, "save host software installation result") + } + + if status := result.Status(); status != fleet.SoftwareInstallerPending { + hsi, err := svc.ds.GetSoftwareInstallResults(ctx, result.InstallUUID) + if err != nil { + return ctxerr.Wrap(ctx, err, "get host software installation result information") + } + + var user *fleet.User + if hsi.UserID != nil { + user, err = svc.ds.UserByID(ctx, *hsi.UserID) + if err != nil { + return ctxerr.Wrap(ctx, err, "get host software installation user") + } + } + + if err := svc.ds.NewActivity( + ctx, + user, + fleet.ActivityTypeInstalledSoftware{ + HostID: host.ID, + HostDisplayName: host.DisplayName(), + SoftwareTitle: hsi.SoftwareTitle, + InstallUUID: result.InstallUUID, + Status: string(status), + }, + ); err != nil { + return ctxerr.Wrap(ctx, err, "create activity for software installation") + } + } + return nil } From 37fe905f96961c934fb522ed176fc73143f1d7ef Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Tue, 7 May 2024 13:02:08 -0300 Subject: [PATCH 21/56] missing validations and tweaks to default scripts (#18780) This adds two things: - when implementing the CLI, I found [a panel](https://www.figma.com/file/oQl2oQUG0iRkUy0YOxc307/%2314921-Deploy-security-agents-to-macOS%2C-Windows%2C-and-Linux-hosts?type=design&node-id=779-29335&mode=design&t=Y27cbj7DdhUEGJko-4) in the Figma file with validations that I missed - explicit shebang for bash scrips (requested by product) and removed a comment that will be user facing for exe files. --- ee/server/service/software_installers.go | 40 +++++++- pkg/file/scripts/install_deb.sh | 2 + pkg/file/scripts/install_exe.ps1 | 5 - pkg/file/scripts/install_pkg.sh | 2 + pkg/file/scripts/remove_deb.sh | 2 + pkg/file/scripts/remove_pkg.sh | 2 + .../testdata/scripts/install_deb.sh.golden | 2 + .../testdata/scripts/install_exe.ps1.golden | 5 - .../testdata/scripts/install_pkg.sh.golden | 2 + .../testdata/scripts/remove_deb.sh.golden | 2 + .../testdata/scripts/remove_pkg.sh.golden | 2 + server/datastore/mysql/activities_test.go | 10 +- server/datastore/mysql/software_installers.go | 49 +++++++--- .../mysql/software_installers_test.go | 22 +++-- server/fleet/datastore.go | 5 +- server/mock/datastore_mock.go | 18 +++- server/service/integration_core_test.go | 2 +- server/service/integration_mdm_test.go | 93 ++++++++++++++++++- 18 files changed, 217 insertions(+), 48 deletions(-) diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index f55c3a2555..e20ca36878 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -4,7 +4,9 @@ import ( "context" "encoding/hex" "errors" + "fmt" "net/http" + "path/filepath" "strings" "github.com/fleetdm/fleet/v4/pkg/file" @@ -219,19 +221,47 @@ func (svc *Service) InstallSoftwareTitle(ctx context.Context, hostID uint, softw return err } - err = svc.ds.InsertSoftwareInstallRequest(ctx, hostID, softwareTitleID, host.TeamID) + installer, err := svc.ds.GetSoftwareInstallerForTitle(ctx, softwareTitleID, host.TeamID) if err != nil { if fleet.IsNotFound(err) { return &fleet.BadRequestError{ - Message: "The software title provided doesn't have an installer", - InternalErr: ctxerr.Wrapf(ctx, err, "couldn't find an installer for software title"), + Message: "Software title has no package added. Please add software package to install.", + InternalErr: ctxerr.WrapWithData( + ctx, err, "couldn't find an installer for software title", + map[string]any{"host_id": host.ID, "team_id": host.TeamID, "title_id": softwareTitleID}, + ), } } - return ctxerr.Wrap(ctx, err, "inserting software install request") + return ctxerr.Wrap(ctx, err, "finding software installer for title") } - return nil + ext := filepath.Ext(installer.Name) + var requiredPlatform string + switch ext { + case ".msi", ".exe": + requiredPlatform = "windows" + case ".pkg": + requiredPlatform = "darwin" + case ".deb": + requiredPlatform = "linux" + default: + // this should never happen + return ctxerr.Errorf(ctx, "software installer has unsupported type %s", ext) + } + + if host.FleetPlatform() != requiredPlatform { + return &fleet.BadRequestError{ + Message: fmt.Sprintf("Package (%s) can be installed only on %s hosts.", ext, requiredPlatform), + InternalErr: ctxerr.WrapWithData( + ctx, err, "invalid host platform for requested installer", + map[string]any{"host_id": host.ID, "team_id": host.TeamID, "title_id": softwareTitleID}, + ), + } + } + + err = svc.ds.InsertSoftwareInstallRequest(ctx, hostID, installer.InstallerID) + return ctxerr.Wrap(ctx, err, "inserting software install request") } func (svc *Service) GetSoftwareInstallResults(ctx context.Context, resultUUID string) (*fleet.HostSoftwareInstallerResult, error) { diff --git a/pkg/file/scripts/install_deb.sh b/pkg/file/scripts/install_deb.sh index 16d246663e..78dbeafbe3 100644 --- a/pkg/file/scripts/install_deb.sh +++ b/pkg/file/scripts/install_deb.sh @@ -1 +1,3 @@ +#!/bin/sh + apt-get install -f "$INSTALLER_PATH" diff --git a/pkg/file/scripts/install_exe.ps1 b/pkg/file/scripts/install_exe.ps1 index 4aa91f5b1c..e205c4ed11 100644 --- a/pkg/file/scripts/install_exe.ps1 +++ b/pkg/file/scripts/install_exe.ps1 @@ -4,11 +4,6 @@ $exeFilePath = "$INSTALLER_PATH" $exeName = [System.IO.Path]::GetFileName($exeFilePath) $subDir = [System.IO.Path]::GetFileNameWithoutExtension($exeFilePath) -# Program Files is the recommended location for any third-party software on Windows. -# -# Note: a x86 binary on a x64 system is supposed to go in -# $env:ProgramFiles(x86) but I didn't find a reliable way to get this -# information from the exe file. $destinationPath = Join-Path -Path $env:ProgramFiles -ChildPath $subDir # check if the directory does not exist, and create it if necessary diff --git a/pkg/file/scripts/install_pkg.sh b/pkg/file/scripts/install_pkg.sh index 783d2941a7..837c32d15b 100644 --- a/pkg/file/scripts/install_pkg.sh +++ b/pkg/file/scripts/install_pkg.sh @@ -1 +1,3 @@ +#!/bin/sh + installer -pkg "$INSTALLER_PATH" -target / diff --git a/pkg/file/scripts/remove_deb.sh b/pkg/file/scripts/remove_deb.sh index 5de7123909..1a7c01d3e1 100644 --- a/pkg/file/scripts/remove_deb.sh +++ b/pkg/file/scripts/remove_deb.sh @@ -1 +1,3 @@ +#!/bin/sh + apt-get remove -y $(dpkg -f "$INSTALLER_PATH" Package) diff --git a/pkg/file/scripts/remove_pkg.sh b/pkg/file/scripts/remove_pkg.sh index 84ceb7a3fc..596a61166e 100644 --- a/pkg/file/scripts/remove_pkg.sh +++ b/pkg/file/scripts/remove_pkg.sh @@ -1,3 +1,5 @@ +#!/bin/sh + # grab the identifier from the first PackageInfo we find. Those are placed in different locations depending on the installer pkg_id=$(tar xOvf "$INSTALLER_PATH" --include='*PackageInfo*' 2>/dev/null | sed -n 's/.*identifier="\([^"]*\)".*/\1/p') diff --git a/pkg/file/testdata/scripts/install_deb.sh.golden b/pkg/file/testdata/scripts/install_deb.sh.golden index 16d246663e..78dbeafbe3 100644 --- a/pkg/file/testdata/scripts/install_deb.sh.golden +++ b/pkg/file/testdata/scripts/install_deb.sh.golden @@ -1 +1,3 @@ +#!/bin/sh + apt-get install -f "$INSTALLER_PATH" diff --git a/pkg/file/testdata/scripts/install_exe.ps1.golden b/pkg/file/testdata/scripts/install_exe.ps1.golden index 4aa91f5b1c..e205c4ed11 100644 --- a/pkg/file/testdata/scripts/install_exe.ps1.golden +++ b/pkg/file/testdata/scripts/install_exe.ps1.golden @@ -4,11 +4,6 @@ $exeFilePath = "$INSTALLER_PATH" $exeName = [System.IO.Path]::GetFileName($exeFilePath) $subDir = [System.IO.Path]::GetFileNameWithoutExtension($exeFilePath) -# Program Files is the recommended location for any third-party software on Windows. -# -# Note: a x86 binary on a x64 system is supposed to go in -# $env:ProgramFiles(x86) but I didn't find a reliable way to get this -# information from the exe file. $destinationPath = Join-Path -Path $env:ProgramFiles -ChildPath $subDir # check if the directory does not exist, and create it if necessary diff --git a/pkg/file/testdata/scripts/install_pkg.sh.golden b/pkg/file/testdata/scripts/install_pkg.sh.golden index 783d2941a7..837c32d15b 100644 --- a/pkg/file/testdata/scripts/install_pkg.sh.golden +++ b/pkg/file/testdata/scripts/install_pkg.sh.golden @@ -1 +1,3 @@ +#!/bin/sh + installer -pkg "$INSTALLER_PATH" -target / diff --git a/pkg/file/testdata/scripts/remove_deb.sh.golden b/pkg/file/testdata/scripts/remove_deb.sh.golden index 5de7123909..1a7c01d3e1 100644 --- a/pkg/file/testdata/scripts/remove_deb.sh.golden +++ b/pkg/file/testdata/scripts/remove_deb.sh.golden @@ -1 +1,3 @@ +#!/bin/sh + apt-get remove -y $(dpkg -f "$INSTALLER_PATH" Package) diff --git a/pkg/file/testdata/scripts/remove_pkg.sh.golden b/pkg/file/testdata/scripts/remove_pkg.sh.golden index 84ceb7a3fc..596a61166e 100644 --- a/pkg/file/testdata/scripts/remove_pkg.sh.golden +++ b/pkg/file/testdata/scripts/remove_pkg.sh.golden @@ -1,3 +1,5 @@ +#!/bin/sh + # grab the identifier from the first PackageInfo we find. Those are placed in different locations depending on the installer pkg_id=$(tar xOvf "$INSTALLER_PATH" --include='*PackageInfo*' 2>/dev/null | sed -n 's/.*identifier="\([^"]*\)".*/\1/p') diff --git a/server/datastore/mysql/activities_test.go b/server/datastore/mysql/activities_test.go index 40f2cd061a..d2d252f473 100644 --- a/server/datastore/mysql/activities_test.go +++ b/server/datastore/mysql/activities_test.go @@ -382,10 +382,10 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { require.NoError(t, err) h1E := hsr.ExecutionID // create some software installs requests for h1, make some complete - err = ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1Meta.TitleID, nil) + err = ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1Meta.InstallerID) require.NoError(t, err) h1FooFailed := latestSoftwareInstallerUUID() - err = ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw2Meta.TitleID, nil) + err = ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw2Meta.InstallerID) require.NoError(t, err) h1Bar := latestSoftwareInstallerUUID() err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ @@ -394,7 +394,7 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { PreInstallConditionOutput: ptr.String(""), // pre-install failed }) require.NoError(t, err) - err = ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1Meta.TitleID, nil) + err = ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1Meta.InstallerID) require.NoError(t, err) h1FooInstalled := latestSoftwareInstallerUUID() err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ @@ -404,7 +404,7 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { InstallScriptExitCode: ptr.Int(0), }) require.NoError(t, err) - err = ds.InsertSoftwareInstallRequest(noUserCtx, h1.ID, sw1Meta.TitleID, nil) // no user for this one + err = ds.InsertSoftwareInstallRequest(noUserCtx, h1.ID, sw1Meta.InstallerID) // no user for this one require.NoError(t, err) h1Foo := latestSoftwareInstallerUUID() @@ -418,7 +418,7 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { require.NoError(t, err) h2F := hsr.ExecutionID // add a pending software install request for h2 - err = ds.InsertSoftwareInstallRequest(ctx, h2.ID, sw2Meta.TitleID, nil) + err = ds.InsertSoftwareInstallRequest(ctx, h2.ID, sw2Meta.InstallerID) require.NoError(t, err) h2Bar := latestSoftwareInstallerUUID() diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index f9d7f6a5ee..0937e859cb 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -192,12 +192,43 @@ func (ds *Datastore) DeleteSoftwareInstaller(ctx context.Context, id uint) error return nil } -func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareTitleID uint, teamID *uint) error { +func (ds *Datastore) GetSoftwareInstallerForTitle(ctx context.Context, softwareTitleID uint, teamID *uint) (*fleet.SoftwareInstaller, error) { var tmID uint if teamID != nil { tmID = *teamID } + const getInstallerIDStmt = ` +SELECT + id, + team_id, + title_id, + storage_id, + filename, + version, + install_script_content_id, + pre_install_query, + post_install_script_content_id, + uploaded_at +FROM + software_installers +WHERE + title_id = ? AND global_or_team_id = ?` + + var installer fleet.SoftwareInstaller + err := sqlx.GetContext(ctx, ds.reader(ctx), &installer, getInstallerIDStmt, softwareTitleID, tmID) + if err != nil { + if err == sql.ErrNoRows { + return nil, notFound("SoftwareInstaller") + } + + return nil, ctxerr.Wrap(ctx, err, "finding software installer by title") + } + + return &installer, nil +} + +func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareInstallerID uint) error { const ( insertStmt = ` INSERT INTO host_software_installs @@ -206,8 +237,6 @@ func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID ui (?, ?, ?, ?) ` - getInstallerIDStmt = `SELECT id FROM software_installers WHERE title_id = ? AND global_or_team_id = ?` - hostExistsStmt = `SELECT 1 FROM hosts WHERE id = ?` ) @@ -219,17 +248,7 @@ func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID ui return notFound("Host").WithID(hostID) } - return ctxerr.Wrap(ctx, err, "inserting new install software request") - } - - var installerID uint - err = sqlx.GetContext(ctx, ds.reader(ctx), &installerID, getInstallerIDStmt, softwareTitleID, tmID) - if err != nil { - if err == sql.ErrNoRows { - return notFound("SoftwareInstaller") - } - - return ctxerr.Wrap(ctx, err, "inserting new install software request") + return ctxerr.Wrap(ctx, err, "checking if host exists") } var userID *uint @@ -239,7 +258,7 @@ func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID ui _, err = ds.writer(ctx).ExecContext(ctx, insertStmt, uuid.NewString(), hostID, - installerID, + softwareInstallerID, userID, ) diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go index a0e9b91cec..bf3c84ac1c 100644 --- a/server/datastore/mysql/software_installers_test.go +++ b/server/datastore/mysql/software_installers_test.go @@ -22,8 +22,8 @@ func TestSoftwareInstallers(t *testing.T) { name string fn func(t *testing.T, ds *Datastore) }{ + {"SoftwareInstallRequests", testSoftwareInstallRequests}, {"SoftwareInstallerDetails", testListSoftwareInstallerDetails}, - {"InsertSoftwareInstallRequest", testInsertSoftwareInstallRequest}, {"GetSoftwareInstallResults", testGetSoftwareInstallResult}, } @@ -171,7 +171,7 @@ func insertSoftwareInstaller( return res, nil } -func testInsertSoftwareInstallRequest(t *testing.T, ds *Datastore) { +func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { ctx := context.Background() // create a team @@ -185,23 +185,31 @@ func testInsertSoftwareInstallRequest(t *testing.T, ds *Datastore) { for tc, teamID := range cases { t.Run(tc, func(t *testing.T) { - // non-existent installer and host does the installer check first - err := ds.InsertSoftwareInstallRequest(ctx, 1, 1, teamID) + // non-existent installer + si, err := ds.GetSoftwareInstallerForTitle(ctx, 1, teamID) var nfe fleet.NotFoundError require.ErrorAs(t, err, &nfe) + require.Nil(t, si) - // non-existent host installerID, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ Title: "foo", Source: "bar", InstallScript: "echo", TeamID: teamID, + Filename: "foo.pkg", }) require.NoError(t, err) + installerMeta, err := ds.GetSoftwareInstallerMetadata(ctx, installerID) require.NoError(t, err) - err = ds.InsertSoftwareInstallRequest(ctx, 12, installerMeta.TitleID, teamID) + si, err = ds.GetSoftwareInstallerForTitle(ctx, installerMeta.TitleID, teamID) + require.NoError(t, err) + require.NotNil(t, si) + require.Equal(t, "foo.pkg", si.Name) + + // non-existent host + err = ds.InsertSoftwareInstallRequest(ctx, 12, si.InstallerID) require.ErrorAs(t, err, &nfe) // successful insert @@ -214,7 +222,7 @@ func testInsertSoftwareInstallRequest(t *testing.T, ds *Datastore) { TeamID: teamID, }) require.NoError(t, err) - err = ds.InsertSoftwareInstallRequest(ctx, host.ID, installerMeta.TitleID, teamID) + err = ds.InsertSoftwareInstallRequest(ctx, host.ID, si.InstallerID) require.NoError(t, err) }) } diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 47689e340d..33d3198097 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -491,7 +491,10 @@ type Datastore interface { SoftwareTitleByID(ctx context.Context, id uint, teamID *uint, tmFilter TeamFilter) (*SoftwareTitle, error) // InsertSoftwareInstallRequest tracks a new request to install the provided software installer in the host - InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareInstallerID uint, teamID *uint) error + InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareTitleID uint) error + + // GetSoftwareInstallerForTitle TODO + GetSoftwareInstallerForTitle(ctx context.Context, softwareTitleID uint, teamID *uint) (*SoftwareInstaller, error) /////////////////////////////////////////////////////////////////////////////// // SoftwareStore diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index fe96f32527..3abc80fb7a 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -365,7 +365,9 @@ type ListSoftwareTitlesFunc func(ctx context.Context, opt fleet.SoftwareTitleLis type SoftwareTitleByIDFunc func(ctx context.Context, id uint, teamID *uint, tmFilter fleet.TeamFilter) (*fleet.SoftwareTitle, error) -type InsertSoftwareInstallRequestFunc func(ctx context.Context, hostID uint, softwareInstallerID uint, teamID *uint) error +type InsertSoftwareInstallRequestFunc func(ctx context.Context, hostID uint, softwareTitleID uint) error + +type GetSoftwareInstallerForTitleFunc func(ctx context.Context, softwareTitleID uint, teamID *uint) (*fleet.SoftwareInstaller, error) type ListSoftwareForVulnDetectionFunc func(ctx context.Context, hostID uint) ([]fleet.Software, error) @@ -1460,6 +1462,9 @@ type DataStore struct { InsertSoftwareInstallRequestFunc InsertSoftwareInstallRequestFunc InsertSoftwareInstallRequestFuncInvoked bool + GetSoftwareInstallerForTitleFunc GetSoftwareInstallerForTitleFunc + GetSoftwareInstallerForTitleFuncInvoked bool + ListSoftwareForVulnDetectionFunc ListSoftwareForVulnDetectionFunc ListSoftwareForVulnDetectionFuncInvoked bool @@ -3529,11 +3534,18 @@ func (s *DataStore) SoftwareTitleByID(ctx context.Context, id uint, teamID *uint return s.SoftwareTitleByIDFunc(ctx, id, teamID, tmFilter) } -func (s *DataStore) InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareInstallerID uint, teamID *uint) error { +func (s *DataStore) InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareTitleID uint) error { s.mu.Lock() s.InsertSoftwareInstallRequestFuncInvoked = true s.mu.Unlock() - return s.InsertSoftwareInstallRequestFunc(ctx, hostID, softwareInstallerID, teamID) + return s.InsertSoftwareInstallRequestFunc(ctx, hostID, softwareTitleID) +} + +func (s *DataStore) GetSoftwareInstallerForTitle(ctx context.Context, softwareTitleID uint, teamID *uint) (*fleet.SoftwareInstaller, error) { + s.mu.Lock() + s.GetSoftwareInstallerForTitleFuncInvoked = true + s.mu.Unlock() + return s.GetSoftwareInstallerForTitleFunc(ctx, softwareTitleID, teamID) } func (s *DataStore) ListSoftwareForVulnDetection(ctx context.Context, hostID uint) ([]fleet.Software, error) { diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index 1e9c7b1a43..edf03091c1 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -11141,7 +11141,7 @@ func (s *integrationTestSuite) TestListHostUpcomingActivities() { require.NoError(t, err) s1Meta, err := s.ds.GetSoftwareInstallerMetadata(ctx, sw1) require.NoError(t, err) - err = s.ds.InsertSoftwareInstallRequest(ctx, host1.ID, s1Meta.TitleID, nil) + err = s.ds.InsertSoftwareInstallRequest(ctx, host1.ID, s1Meta.InstallerID) require.NoError(t, err) h1Foo := latestSoftwareInstallerUUID() diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 1b3fcd0c1a..078749e83c 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -6,6 +6,7 @@ import ( "crypto/x509" "database/sql" "encoding/base64" + "encoding/hex" "encoding/json" "encoding/xml" "errors" @@ -8637,6 +8638,96 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadDownloadAndDelete() }) } +func (s *integrationMDMTestSuite) TestSoftwareInstallerNewInstallRequestPlatformValidation() { + t := s.T() + + hostsByPlatform := map[string]*fleet.Host{ + "linux": nil, "darwin": nil, "windows": nil, + } + + tm, err := s.ds.NewTeam(context.Background(), &fleet.Team{ + Name: t.Name(), + Description: "desc", + }) + require.NoError(t, err) + + for platform := range hostsByPlatform { + h, err := s.ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now().Add(-1 * time.Minute), + OsqueryHostID: ptr.String(t.Name() + uuid.New().String()), + NodeKey: ptr.String(t.Name() + uuid.New().String()), + Hostname: fmt.Sprintf("%sfoo.local", t.Name()), + Platform: platform, + }) + require.NoError(t, err) + setOrbitEnrollment(t, h, s.ds) + + err = s.ds.AddHostsToTeam(context.Background(), &tm.ID, []uint{h.ID}) + require.NoError(t, err) + + hostsByPlatform[platform] = h + } + + softwareTitles := map[string]uint{ + "deb": 0, "msi": 0, "exe": 0, "pkg": 0, + } + + for kind := range softwareTitles { + // TODO(roberto): we need real binaries for exe, msi and pkg to + // perform the API calls. + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + ctx := context.Background() + installScript := fmt.Sprintf(`echo '%s'`, kind) + res, err := q.ExecContext(ctx, `INSERT INTO script_contents (md5_checksum, contents) VALUES (UNHEX(md5(?)), ?)`, installScript, installScript) + if err != nil { + return err + } + scriptContentID, _ := res.LastInsertId() + + res, err = q.ExecContext(ctx, `INSERT INTO software_titles (name, source) VALUES ('foo', ?)`, kind) + if err != nil { + return err + } + titleID, _ := res.LastInsertId() + softwareTitles[kind] = uint(titleID) + + _, err = q.ExecContext(ctx, ` + INSERT INTO software_installers + (title_id, filename, version, install_script_content_id, storage_id, team_id, global_or_team_id, pre_install_query) + VALUES + (?, ?, ?, ?, unhex(?), ?, ?, ?)`, + titleID, fmt.Sprintf("installer.%s", kind), "v1.0.0", scriptContentID, hex.EncodeToString([]byte("test")), tm.ID, tm.ID, "foo") + return err + }) + } + + testCases := []struct { + platform string + supportedInstallers []string + }{ + {"windows", []string{"exe", "msi"}}, + {"darwin", []string{"pkg"}}, + {"linux", []string{"deb"}}, + } + + for _, tc := range testCases { + for platform, host := range hostsByPlatform { + for _, kind := range tc.supportedInstallers { + wantStatus := http.StatusAccepted + if tc.platform != platform { + wantStatus = http.StatusBadRequest + } + + var resp installSoftwareResponse + s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/hosts/%d/software/install/%d", host.ID, softwareTitles[kind]), nil, wantStatus, &resp) + } + } + } +} + func (s *integrationMDMTestSuite) TestSoftwareInstallerNewInstallRequest() { t := s.T() @@ -8653,7 +8744,7 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerNewInstallRequest() { OsqueryHostID: ptr.String(t.Name() + uuid.New().String()), NodeKey: ptr.String(t.Name() + uuid.New().String()), Hostname: fmt.Sprintf("%sfoo.local", t.Name()), - Platform: "darwin", + Platform: "linux", }) require.NoError(t, err) From 4f9363fd7891bfe0b8e7287a00ea81e0c5497871 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Tue, 7 May 2024 16:50:44 -0400 Subject: [PATCH 22/56] Add cron job to cleanup unused software installers (#18812) --- .../18673-cleanup-unused-software-installers | 1 + cmd/fleet/cron.go | 4 ++ cmd/fleet/serve.go | 4 +- .../filesystem/software_installer.go | 32 ++++++++++ .../filesystem/software_installer_test.go | 61 +++++++++++++++++++ ...240424124712_AddSoftwareInstallerTables.go | 2 +- server/datastore/mysql/schema.sql | 2 +- server/datastore/mysql/software_installers.go | 16 +++++ .../mysql/software_installers_test.go | 60 ++++++++++++++++++ server/datastore/s3/software_installer.go | 47 ++++++++++++++ .../datastore/s3/software_installer_test.go | 61 +++++++++++++++++++ server/fleet/datastore.go | 4 ++ server/fleet/software_installer.go | 8 +++ server/mock/datastore_mock.go | 12 ++++ 14 files changed, 310 insertions(+), 4 deletions(-) create mode 100644 changes/18673-cleanup-unused-software-installers diff --git a/changes/18673-cleanup-unused-software-installers b/changes/18673-cleanup-unused-software-installers new file mode 100644 index 0000000000..1e39a89f2d --- /dev/null +++ b/changes/18673-cleanup-unused-software-installers @@ -0,0 +1 @@ +* Added a `cron` job to periodically remove unused software installers from the store. diff --git a/cmd/fleet/cron.go b/cmd/fleet/cron.go index 824d09e515..4b8d65930c 100644 --- a/cmd/fleet/cron.go +++ b/cmd/fleet/cron.go @@ -708,6 +708,7 @@ func newCleanupsAndAggregationSchedule( enrollHostLimiter fleet.EnrollHostLimiter, config *config.FleetConfig, commander *apple_mdm.MDMAppleCommander, + softwareInstallStore fleet.SoftwareInstallerStore, ) (*schedule.Schedule, error) { const ( name = string(fleet.CronCleanupsThenAggregation) @@ -848,6 +849,9 @@ func newCleanupsAndAggregationSchedule( const maxCount = 5000 return ds.CleanupActivitiesAndAssociatedData(ctx, maxCount, appConfig.ActivityExpirySettings.ActivityExpiryWindow) }), + schedule.WithJob("cleanup_unused_software_installers", func(ctx context.Context) error { + return ds.CleanupUnusedSoftwareInstallers(ctx, softwareInstallStore) + }), ) return s, nil diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 108d568707..cd6c98244c 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -627,12 +627,12 @@ the way that the Fleet server works. initFatal(err, "initializing service") } + var softwareInstallStore fleet.SoftwareInstallerStore if license.IsPremium() { var profileMatcher fleet.ProfileMatcher if appCfg.MDM.EnabledAndConfigured { profileMatcher = apple_mdm.NewProfileMatcher(redisPool) } - var softwareInstallStore fleet.SoftwareInstallerStore if config.S3.Bucket != "" { store, err := s3.NewSoftwareInstallerStore(config.S3) if err != nil { @@ -724,7 +724,7 @@ the way that the Fleet server works. commander = apple_mdm.NewMDMAppleCommander(mdmStorage, mdmPushService, config.MDM) } return newCleanupsAndAggregationSchedule( - ctx, instanceID, ds, logger, redisWrapperDS, &config, commander, + ctx, instanceID, ds, logger, redisWrapperDS, &config, commander, softwareInstallStore, ) }, ); err != nil { diff --git a/server/datastore/filesystem/software_installer.go b/server/datastore/filesystem/software_installer.go index e8d297fad9..7bb68a87cc 100644 --- a/server/datastore/filesystem/software_installer.go +++ b/server/datastore/filesystem/software_installer.go @@ -2,6 +2,7 @@ package filesystem import ( "context" + "errors" "io" "os" "path/filepath" @@ -92,6 +93,37 @@ func (i *SoftwareInstallerStore) Exists(ctx context.Context, installerID string) return true, nil } +func (i *SoftwareInstallerStore) Cleanup(ctx context.Context, usedInstallerIDs []string) (int, error) { + usedSet := make(map[string]struct{}, len(usedInstallerIDs)) + for _, id := range usedInstallerIDs { + usedSet[id] = struct{}{} + } + + baseDir := filepath.Join(i.rootDir, softwareInstallersPrefix) + dirEnts, err := os.ReadDir(baseDir) + if err != nil { + return 0, ctxerr.Wrap(ctx, err, "listing software installers in filesystem store") + } + + // collect deletion errors so that it keeps going if possible + var errs []error + var count int + for _, de := range dirEnts { + if !de.Type().IsRegular() { + continue + } + if _, isUsed := usedSet[de.Name()]; isUsed { + continue + } + if err := os.Remove(filepath.Join(baseDir, de.Name())); err != nil { + errs = append(errs, err) + } else { + count++ + } + } + return count, ctxerr.Wrap(ctx, errors.Join(errs...), "delete unused software installers") +} + // pathForInstaller builds local filesystem path to identify the software // installer. func (i *SoftwareInstallerStore) pathForInstaller(installerID string) string { diff --git a/server/datastore/filesystem/software_installer_test.go b/server/datastore/filesystem/software_installer_test.go index e3197cbf74..3b942df885 100644 --- a/server/datastore/filesystem/software_installer_test.go +++ b/server/datastore/filesystem/software_installer_test.go @@ -6,10 +6,14 @@ import ( "crypto/rand" "crypto/sha256" "encoding/hex" + "fmt" "io" + "os" + "path/filepath" "testing" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/google/uuid" "github.com/stretchr/testify/require" ) @@ -82,3 +86,60 @@ func TestSoftwareInstaller(t *testing.T) { // read it back, it should still match getAndCheck(id0, b0) } + +func TestSoftwareInstallerCleanup(t *testing.T) { + ctx := context.Background() + + dir := t.TempDir() + store, err := NewSoftwareInstallerStore(dir) + require.NoError(t, err) + + assertExisting := func(want []string) { + dirEnts, err := os.ReadDir(filepath.Join(dir, softwareInstallersPrefix)) + require.NoError(t, err) + got := make([]string, 0, len(dirEnts)) + for _, de := range dirEnts { + if de.Type().IsRegular() { + got = append(got, de.Name()) + } + } + require.ElementsMatch(t, want, got) + } + + // cleanup an empty store + n, err := store.Cleanup(ctx, nil) + require.NoError(t, err) + require.Equal(t, 0, n) + + // put an installer + ins0 := uuid.NewString() + err = store.Put(ctx, ins0, bytes.NewReader([]byte("installer0"))) + require.NoError(t, err) + + // cleanup but mark it as used + n, err = store.Cleanup(ctx, []string{ins0}) + require.NoError(t, err) + require.Equal(t, 0, n) + + assertExisting([]string{ins0}) + + // cleanup but mark it as unused + n, err = store.Cleanup(ctx, []string{}) + require.NoError(t, err) + require.Equal(t, 1, n) + + assertExisting(nil) + + // put a few installers + installers := []string{uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString()} + for i, ins := range installers { + err = store.Put(ctx, ins, bytes.NewReader([]byte("installer"+fmt.Sprint(i)))) + require.NoError(t, err) + } + + n, err = store.Cleanup(ctx, []string{installers[0], installers[2]}) + require.NoError(t, err) + require.Equal(t, 2, n) + + assertExisting([]string{installers[0], installers[2]}) +} diff --git a/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go b/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go index 478aa8b0f0..326487b371 100644 --- a/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go +++ b/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go @@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS software_installers ( post_install_script_content_id int(10) unsigned DEFAULT NULL, -- used to track the ID retrieved from the storage containing the installer bytes - storage_id binary(64) NOT NULL, + storage_id varchar(64) COLLATE utf8mb4_unicode_ci NOT NULL, uploaded_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 2cd4865f2a..2e6059b14c 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -1510,7 +1510,7 @@ CREATE TABLE `software_installers` ( `pre_install_query` text COLLATE utf8mb4_unicode_ci, `install_script_content_id` int(10) unsigned NOT NULL, `post_install_script_content_id` int(10) unsigned DEFAULT NULL, - `storage_id` binary(64) NOT NULL, + `storage_id` varchar(64) COLLATE utf8mb4_unicode_ci NOT NULL, `uploaded_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `idx_software_installers_team_id_title_id` (`global_or_team_id`,`title_id`), diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 0937e859cb..d23253350f 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -313,3 +313,19 @@ WHERE return &dest, nil } + +func (ds *Datastore) CleanupUnusedSoftwareInstallers(ctx context.Context, softwareInstallStore fleet.SoftwareInstallerStore) error { + if softwareInstallStore == nil { + // no-op in this case, possible if not running with a Premium license + return nil + } + + // get the list of software installers hashes that are in use + var storageIDs []string + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &storageIDs, `SELECT DISTINCT storage_id FROM software_installers`); err != nil { + return ctxerr.Wrap(ctx, err, "get list of software installers in use") + } + + _, err := softwareInstallStore.Cleanup(ctx, storageIDs) + return ctxerr.Wrap(ctx, err, "cleanup unused software installers") +} diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go index bf3c84ac1c..6a7f7be871 100644 --- a/server/datastore/mysql/software_installers_test.go +++ b/server/datastore/mysql/software_installers_test.go @@ -1,12 +1,16 @@ package mysql import ( + "bytes" "context" "database/sql" + "os" + "path/filepath" "testing" "time" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/datastore/filesystem" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/ptr" "github.com/fleetdm/fleet/v4/server/test" @@ -25,6 +29,7 @@ func TestSoftwareInstallers(t *testing.T) { {"SoftwareInstallRequests", testSoftwareInstallRequests}, {"SoftwareInstallerDetails", testListSoftwareInstallerDetails}, {"GetSoftwareInstallResults", testGetSoftwareInstallResult}, + {"CleanupUnusedSoftwareInstallers", testCleanupUnusedSoftwareInstallers}, } for _, c := range cases { @@ -331,3 +336,58 @@ func testGetSoftwareInstallResult(t *testing.T, ds *Datastore) { }) } } + +func testCleanupUnusedSoftwareInstallers(t *testing.T, ds *Datastore) { + ctx := context.Background() + + dir := t.TempDir() + store, err := filesystem.NewSoftwareInstallerStore(dir) + require.NoError(t, err) + + assertExisting := func(want []string) { + dirEnts, err := os.ReadDir(filepath.Join(dir, "software-installers")) + require.NoError(t, err) + got := make([]string, 0, len(dirEnts)) + for _, de := range dirEnts { + if de.Type().IsRegular() { + got = append(got, de.Name()) + } + } + require.ElementsMatch(t, want, got) + } + + // cleanup an empty store + err = ds.CleanupUnusedSoftwareInstallers(ctx, store) + require.NoError(t, err) + assertExisting(nil) + + // put an installer and save it in the DB + ins0 := "installer0" + ins0File := bytes.NewReader([]byte("installer0")) + err = store.Put(ctx, ins0, ins0File) + require.NoError(t, err) + assertExisting([]string{ins0}) + + swi, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "install", + InstallerFile: ins0File, + StorageID: ins0, + Filename: "installer0", + Title: "ins0", + Source: "apps", + }) + require.NoError(t, err) + + assertExisting([]string{ins0}) + err = ds.CleanupUnusedSoftwareInstallers(ctx, store) + require.NoError(t, err) + assertExisting([]string{ins0}) + + // remove it from the DB, will now cleanup + err = ds.DeleteSoftwareInstaller(ctx, swi) + require.NoError(t, err) + + err = ds.CleanupUnusedSoftwareInstallers(ctx, store) + require.NoError(t, err) + assertExisting(nil) +} diff --git a/server/datastore/s3/software_installer.go b/server/datastore/s3/software_installer.go index 8b3d7462cb..1966326266 100644 --- a/server/datastore/s3/software_installer.go +++ b/server/datastore/s3/software_installer.go @@ -74,6 +74,53 @@ func (i *SoftwareInstallerStore) Exists(ctx context.Context, installerID string) return true, nil } +func (i *SoftwareInstallerStore) Cleanup(ctx context.Context, usedInstallerIDs []string) (int, error) { + usedSet := make(map[string]struct{}, len(usedInstallerIDs)) + for _, id := range usedInstallerIDs { + usedSet[id] = struct{}{} + } + + // ListObjectsV2 defaults to a max of 1000 keys, which is sufficient for the + // cleanup task - if more software installers are present, the next run will + // get another 1000 and will periodically complete the cleanups. + // + // Iterating over all pages would potentially take a long time and would make + // it more likely that a conflict arises, where an unused software installer + // becomes used again. This approach makes it only two API requests between + // the read of used installers and the deletions. + prefix := path.Join(i.prefix, softwareInstallersPrefix) + page, err := i.s3client.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: &i.bucket, + Prefix: &prefix, + }) + if err != nil { + return 0, ctxerr.Wrap(ctx, err, "listing software installers in S3 store") + } + + var toDeleteKeys []*s3.ObjectIdentifier + for _, item := range page.Contents { + if item.Key == nil { + continue + } + if _, ok := usedSet[path.Base(*item.Key)]; ok { + continue + } + toDeleteKeys = append(toDeleteKeys, &s3.ObjectIdentifier{Key: item.Key}) + } + + if len(toDeleteKeys) == 0 { + return 0, nil + } + + res, err := i.s3client.DeleteObjects(&s3.DeleteObjectsInput{ + Bucket: &i.bucket, + Delete: &s3.Delete{ + Objects: toDeleteKeys, + }, + }) + return len(res.Deleted), ctxerr.Wrap(ctx, err, "deleting software installers in S3 store") +} + // keyForInstaller builds an S3 key to identify the software installer. func (i *SoftwareInstallerStore) keyForInstaller(installerID string) string { return path.Join(i.prefix, softwareInstallersPrefix, installerID) diff --git a/server/datastore/s3/software_installer_test.go b/server/datastore/s3/software_installer_test.go index 4fb846086e..9292a6ed7a 100644 --- a/server/datastore/s3/software_installer_test.go +++ b/server/datastore/s3/software_installer_test.go @@ -6,10 +6,14 @@ import ( "crypto/rand" "crypto/sha256" "encoding/hex" + "fmt" "io" + "path" "testing" + "github.com/aws/aws-sdk-go/service/s3" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/google/uuid" "github.com/stretchr/testify/require" ) @@ -79,3 +83,60 @@ func TestSoftwareInstaller(t *testing.T) { // read it back, it should still match getAndCheck(id0, b0) } + +func TestSoftwareInstallerCleanup(t *testing.T) { + ctx := context.Background() + store := SetupTestSoftwareInstallerStore(t, "software-installers-unit-test", "prefix") + + assertExisting := func(want []string) { + prefix := path.Join(store.prefix, softwareInstallersPrefix) + page, err := store.s3client.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: &store.bucket, + Prefix: &prefix, + }) + require.NoError(t, err) + + got := make([]string, 0, len(page.Contents)) + for _, item := range page.Contents { + got = append(got, path.Base(*item.Key)) + } + require.ElementsMatch(t, want, got) + } + + // cleanup an empty store + n, err := store.Cleanup(ctx, nil) + require.NoError(t, err) + require.Equal(t, 0, n) + + // put an installer + ins0 := uuid.NewString() + err = store.Put(ctx, ins0, bytes.NewReader([]byte("installer0"))) + require.NoError(t, err) + + // cleanup but mark it as used + n, err = store.Cleanup(ctx, []string{ins0}) + require.NoError(t, err) + require.Equal(t, 0, n) + + assertExisting([]string{ins0}) + + // cleanup but mark it as unused + n, err = store.Cleanup(ctx, []string{}) + require.NoError(t, err) + require.Equal(t, 1, n) + + assertExisting(nil) + + // put a few installers + installers := []string{uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString()} + for i, ins := range installers { + err = store.Put(ctx, ins, bytes.NewReader([]byte("installer"+fmt.Sprint(i)))) + require.NoError(t, err) + } + + n, err = store.Cleanup(ctx, []string{installers[0], installers[2]}) + require.NoError(t, err) + require.Equal(t, 2, n) + + assertExisting([]string{installers[0], installers[2]}) +} diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 7b65a6c35b..23408ca269 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1491,6 +1491,10 @@ type Datastore interface { DeleteSoftwareInstaller(ctx context.Context, id uint) error GetSoftwareInstallResults(ctx context.Context, resultsUUID string) (*HostSoftwareInstallerResult, error) + + // CleanupUnusedSoftwareInstallers will remove software installers that have + // no references to them from the software_installers table. + CleanupUnusedSoftwareInstallers(ctx context.Context, softwareInstallStore SoftwareInstallerStore) error } // MDMAppleStore wraps nanomdm's storage and adds methods to deal with diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index f4f188610d..6ee0dcf87e 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -16,6 +16,7 @@ type SoftwareInstallerStore interface { Get(ctx context.Context, installerID string) (io.ReadCloser, int64, error) Put(ctx context.Context, installerID string, content io.ReadSeeker) error Exists(ctx context.Context, installerID string) (bool, error) + Cleanup(ctx context.Context, usedInstallerIDs []string) (int, error) } // FailingSoftwareInstallerStore is an implementation of SoftwareInstallerStore @@ -35,6 +36,13 @@ func (FailingSoftwareInstallerStore) Exists(ctx context.Context, installerID str return false, errors.New("software installer store not properly configured") } +func (FailingSoftwareInstallerStore) Cleanup(ctx context.Context, usedInstallerIDs []string) (int, error) { + // do not fail for the failing store's cleanup, as unlike the other store + // methods, this will be called even if software installers are otherwise not + // used (by the cron job). + return 0, nil +} + // SoftwareInstallDetailsResult contains all of the information // required for a client to pull in and install software from the fleet server type SoftwareInstallDetails struct { diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index f7142ba225..3dc89b5976 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -945,6 +945,8 @@ type DeleteSoftwareInstallerFunc func(ctx context.Context, id uint) error type GetSoftwareInstallResultsFunc func(ctx context.Context, resultsUUID string) (*fleet.HostSoftwareInstallerResult, error) +type CleanupUnusedSoftwareInstallersFunc func(ctx context.Context, softwareInstallStore fleet.SoftwareInstallerStore) error + type DataStore struct { HealthCheckFunc HealthCheckFunc HealthCheckFuncInvoked bool @@ -2335,6 +2337,9 @@ type DataStore struct { GetSoftwareInstallResultsFunc GetSoftwareInstallResultsFunc GetSoftwareInstallResultsFuncInvoked bool + CleanupUnusedSoftwareInstallersFunc CleanupUnusedSoftwareInstallersFunc + CleanupUnusedSoftwareInstallersFuncInvoked bool + mu sync.Mutex } @@ -5578,3 +5583,10 @@ func (s *DataStore) GetSoftwareInstallResults(ctx context.Context, resultsUUID s s.mu.Unlock() return s.GetSoftwareInstallResultsFunc(ctx, resultsUUID) } + +func (s *DataStore) CleanupUnusedSoftwareInstallers(ctx context.Context, softwareInstallStore fleet.SoftwareInstallerStore) error { + s.mu.Lock() + s.CleanupUnusedSoftwareInstallersFuncInvoked = true + s.mu.Unlock() + return s.CleanupUnusedSoftwareInstallersFunc(ctx, softwareInstallStore) +} From c88a7cf6b00bf2f8405aa8ce04324b030b4f98aa Mon Sep 17 00:00:00 2001 From: Jahziel Villasana-Espinoza Date: Wed, 8 May 2024 10:08:28 -0400 Subject: [PATCH 23/56] feat: software added and deleted global activities (#18798) > Related issue: #18330 # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [x] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements) - [x] Added/updated tests - [x] Manual QA for all new/changed functionality ## Notes - ~I added an `includeTitle bool` parameter to `ds.GetSoftwareInstallerMetadata`. This allows for the title of the software (from the `software_titles` page) to be fetched in `svc.DeleteSoftwareInstaller` without an additional call to the DB.~ We wound up deciding to just fetch the title every time. --------- Co-authored-by: Martin Angers --- changes/18330-global-activites | 1 + docs/Using Fleet/Audit-logs.md | 50 +++++++++++++++- ee/server/service/software_installers.go | 48 +++++++++++++++ frontend/interfaces/activity.ts | 4 ++ .../ActivityItem/ActivityItem.tsx | 40 +++++++++++++ server/datastore/mysql/software_installers.go | 26 ++++---- .../mysql/software_installers_test.go | 4 +- server/fleet/activities.go | 60 ++++++++++++++++++- server/fleet/software_installer.go | 4 +- server/service/integration_core_test.go | 1 - server/service/integration_mdm_test.go | 14 ++++- server/service/software_installers_test.go | 12 ++++ 12 files changed, 240 insertions(+), 24 deletions(-) create mode 100644 changes/18330-global-activites diff --git a/changes/18330-global-activites b/changes/18330-global-activites new file mode 100644 index 0000000000..0c292425fa --- /dev/null +++ b/changes/18330-global-activites @@ -0,0 +1 @@ +- Adds support to the global activity feed for "Added software" and "Deleted software" actions. \ No newline at end of file diff --git a/docs/Using Fleet/Audit-logs.md b/docs/Using Fleet/Audit-logs.md index eb3338d727..add0005ed7 100644 --- a/docs/Using Fleet/Audit-logs.md +++ b/docs/Using Fleet/Audit-logs.md @@ -280,7 +280,7 @@ This activity contains the following fields: ```json { "team_id": 123, - "team_name": "foo" + "team_name": "Workstations" } ``` @@ -297,7 +297,7 @@ This activity contains the following fields: ```json { "team_id": 123, - "team_name": "foo" + "team_name": "Workstations" } ``` @@ -357,7 +357,7 @@ This activity contains the following fields: ```json { "team_id": 123, - "team_name": "foo", + "team_name": "Workstations", "global": false } ``` @@ -1150,6 +1150,50 @@ This activity contains the following fields: } ``` +## added_software + +Generated when a software installer is uploaded to Fleet. + +This activity contains the following fields: +- "software_title": Name of the software. +- "software_package": Filename of the installer. +- "team_name": Name of the team to which this software was added. `null` if it was added to no team." + +- "team_id": The ID of the team to which this software was added. `null` if it was added to no team. + +#### Example + +```json +{ + "software_title": "Falcon.app", + "software_package": "FalconSensor-6.44.pkg", + "team_name": "Workstations", + "team_id": 123 +} + +``` + +## deleted_software + +Generated when a software installer is deleted from Fleet. + +This activity contains the following fields: +- "software_title": Name of the software. +- "software_package": Filename of the installer. +- "team_name": Name of the team to which this software was added. `null if it was added to no team. +- "team_id": The ID of the team to which this software was added. `null` if it was added to no team. + +#### Example + +```json +{ + "software_title": "Falcon.app", + "software_package": "FalconSensor-6.44.pkg", + "team_name": "Workstations", + "team_id": 123 +} + +``` + diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index e20ca36878..9679bc9732 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -12,6 +12,7 @@ import ( "github.com/fleetdm/fleet/v4/pkg/file" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" hostctx "github.com/fleetdm/fleet/v4/server/contexts/host" + "github.com/fleetdm/fleet/v4/server/contexts/viewer" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/go-kit/kit/log/level" ) @@ -28,6 +29,11 @@ func (svc *Service) UploadSoftwareInstaller(ctx context.Context, payload *fleet. return ctxerr.New(ctx, "installer file is required") } + vc, ok := viewer.FromContext(ctx) + if !ok { + return fleet.ErrNoContext + } + title, vers, hash, err := file.ExtractInstallerMetadata(payload.Filename, payload.InstallerFile) if err != nil { // TODO: confirm error handling @@ -80,6 +86,25 @@ func (svc *Service) UploadSoftwareInstaller(ctx context.Context, payload *fleet. // TODO: QA what breaks when you have a software title with no versions? + var teamName *string + if payload.TeamID != nil { + t, err := svc.ds.Team(ctx, *payload.TeamID) + if err != nil { + return err + } + teamName = &t.Name + } + + // Create activity + if err := svc.ds.NewActivity(ctx, vc.User, fleet.ActivityTypeAddedSoftware{ + SoftwareTitle: title, + SoftwarePackage: payload.Filename, + TeamName: teamName, + TeamID: payload.TeamID, + }); err != nil { + return ctxerr.Wrap(ctx, err, "creating activity for added software") + } + return nil } @@ -104,10 +129,33 @@ func (svc *Service) DeleteSoftwareInstaller(ctx context.Context, id uint) error return err } + vc, ok := viewer.FromContext(ctx) + if !ok { + return fleet.ErrNoContext + } + if err := svc.ds.DeleteSoftwareInstaller(ctx, id); err != nil { return ctxerr.Wrap(ctx, err, "deleting software installer") } + var teamName *string + if meta.TeamID != nil { + t, err := svc.ds.Team(ctx, *meta.TeamID) + if err != nil { + return ctxerr.Wrap(ctx, err, "getting team name for deleted software") + } + teamName = &t.Name + } + + if err := svc.ds.NewActivity(ctx, vc.User, fleet.ActivityTypeDeletedSoftware{ + SoftwareTitle: meta.SoftwareTitle, + SoftwarePackage: meta.Name, + TeamName: teamName, + TeamID: meta.TeamID, + }); err != nil { + return ctxerr.Wrap(ctx, err, "creating activity for deleted software") + } + return nil } diff --git a/frontend/interfaces/activity.ts b/frontend/interfaces/activity.ts index 51f8fe1229..a59e9c3773 100644 --- a/frontend/interfaces/activity.ts +++ b/frontend/interfaces/activity.ts @@ -71,6 +71,8 @@ export enum ActivityType { DeletedDeclarationProfile = "deleted_declaration_profile", EditedDeclarationProfile = "edited_declaration_profile", ResentConfigurationProfile = "resent_configuration_profile", + AddedSoftware = "added_software", + DeletedSoftware = "deleted_software", } // This is a subset of ActivityType that are shown only for the host past activities @@ -133,4 +135,6 @@ export interface IActivityDetails { grace_period_days?: number; stats?: ISchedulableQueryStats; host_id?: number; + software_title?: string; + software_package?: string; } diff --git a/frontend/pages/DashboardPage/cards/ActivityFeed/ActivityItem/ActivityItem.tsx b/frontend/pages/DashboardPage/cards/ActivityFeed/ActivityItem/ActivityItem.tsx index 9827afc2ec..3407f2363e 100644 --- a/frontend/pages/DashboardPage/cards/ActivityFeed/ActivityItem/ActivityItem.tsx +++ b/frontend/pages/DashboardPage/cards/ActivityFeed/ActivityItem/ActivityItem.tsx @@ -820,6 +820,40 @@ const TAGGED_TEMPLATES = { ); }, + addedSoftware: (activity: IActivity) => { + return ( + <> + {" "} + added {activity.details?.software_title} ( + {activity.details?.software_package}) software to{" "} + {activity.details?.team_name ? ( + <> + {" "} + the {activity.details?.team_name} team. + + ) : ( + "no team." + )} + + ); + }, + deletedSoftware: (activity: IActivity) => { + return ( + <> + {" "} + deleted {activity.details?.software_title} ( + {activity.details?.software_package}) software from{" "} + {activity.details?.team_name ? ( + <> + {" "} + the {activity.details?.team_name} team. + + ) : ( + "no team." + )} + + ); + }, }; const getDetail = ( @@ -993,6 +1027,12 @@ const getDetail = ( case ActivityType.ResentConfigurationProfile: { return TAGGED_TEMPLATES.resentConfigProfile(activity); } + case ActivityType.AddedSoftware: { + return TAGGED_TEMPLATES.addedSoftware(activity); + } + case ActivityType.DeletedSoftware: { + return TAGGED_TEMPLATES.deletedSoftware(activity); + } default: { return TAGGED_TEMPLATES.defaultActivityTemplate(activity); diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index d23253350f..3d1e5bf509 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -151,20 +151,22 @@ func (ds *Datastore) getOrGenerateSoftwareInstallerTitleID(ctx context.Context, func (ds *Datastore) GetSoftwareInstallerMetadata(ctx context.Context, id uint) (*fleet.SoftwareInstaller, error) { query := ` SELECT - id, - team_id, - title_id, - storage_id, - filename, - version, - install_script_content_id, - pre_install_query, - post_install_script_content_id, - uploaded_at + si.id, + si.team_id, + si.title_id, + si.storage_id, + si.filename, + si.version, + si.install_script_content_id, + si.pre_install_query, + si.post_install_script_content_id, + si.uploaded_at, + COALESCE(st.name, '') AS software_title FROM - software_installers + software_installers si + LEFT OUTER JOIN software_titles st ON st.id = si.title_id WHERE - id = ?` + si.id = ?` var dest fleet.SoftwareInstaller err := sqlx.GetContext(ctx, ds.reader(ctx), &dest, query, id) diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go index 6a7f7be871..87b2f9a140 100644 --- a/server/datastore/mysql/software_installers_test.go +++ b/server/datastore/mysql/software_installers_test.go @@ -168,7 +168,6 @@ func insertSoftwareInstaller( postInstallScriptId, storageId, ) - if err != nil { return nil, ctxerr.Wrap(ctx, err, "inserting software installer") } @@ -204,11 +203,10 @@ func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { Filename: "foo.pkg", }) require.NoError(t, err) - installerMeta, err := ds.GetSoftwareInstallerMetadata(ctx, installerID) require.NoError(t, err) - si, err = ds.GetSoftwareInstallerForTitle(ctx, installerMeta.TitleID, teamID) + si, err = ds.GetSoftwareInstallerForTitle(ctx, *installerMeta.TitleID, teamID) require.NoError(t, err) require.NotNil(t, si) require.Equal(t, "foo.pkg", si.Name) diff --git a/server/fleet/activities.go b/server/fleet/activities.go index 1eccc63c94..5646802e46 100644 --- a/server/fleet/activities.go +++ b/server/fleet/activities.go @@ -92,6 +92,8 @@ var ActivityDetailsList = []ActivityDetails{ ActivityTypeResentConfigurationProfile{}, ActivityTypeInstalledSoftware{}, + ActivityTypeAddedSoftware{}, + ActivityTypeDeletedSoftware{}, } type ActivityDetails interface { @@ -379,7 +381,7 @@ func (a ActivityTypeCreatedTeam) Documentation() (activity string, details strin - "team_id": unique ID of the created team. - "team_name": the name of the created team.`, `{ "team_id": 123, - "team_name": "foo" + "team_name": "Workstations" }` } @@ -398,7 +400,7 @@ func (a ActivityTypeDeletedTeam) Documentation() (activity string, details strin - "team_id": unique ID of the deleted team. - "team_name": the name of the deleted team.`, `{ "team_id": 123, - "team_name": "foo" + "team_name": "Workstations" }` } @@ -471,7 +473,7 @@ func (a ActivityTypeEditedAgentOptions) Documentation() (activity string, detail - "team_id": unique ID of the team for which the agent options were updated (` + "`null`" + ` if global is true). - "team_name": the name of the team for which the agent options were updated (` + "`null`" + ` if global is true).`, `{ "team_id": 123, - "team_name": "foo", + "team_name": "Workstations", "global": false }` } @@ -1448,6 +1450,58 @@ func (a ActivityTypeInstalledSoftware) Documentation() (activity, details, detai }` } +type ActivityTypeAddedSoftware struct { + SoftwareTitle string `json:"software_title"` + SoftwarePackage string `json:"software_package"` + TeamName *string `json:"team_name"` + TeamID *uint `json:"team_id"` +} + +func (a ActivityTypeAddedSoftware) ActivityName() string { + return "added_software" +} + +func (a ActivityTypeAddedSoftware) Documentation() (string, string, string) { + return `Generated when a software installer is uploaded to Fleet.`, `This activity contains the following fields: +- "software_title": Name of the software. +- "software_package": Filename of the installer. +- "team_name": Name of the team to which this software was added.` + " `null` " + `if it was added to no team." + +- "team_id": The ID of the team to which this software was added.` + " `null` " + `if it was added to no team.`, + `{ + "software_title": "Falcon.app", + "software_package": "FalconSensor-6.44.pkg", + "team_name": "Workstations", + "team_id": 123 +} +` +} + +type ActivityTypeDeletedSoftware struct { + SoftwareTitle string `json:"software_title"` + SoftwarePackage string `json:"software_package"` + TeamName *string `json:"team_name"` + TeamID *uint `json:"team_id"` +} + +func (a ActivityTypeDeletedSoftware) ActivityName() string { + return "deleted_software" +} + +func (a ActivityTypeDeletedSoftware) Documentation() (string, string, string) { + return `Generated when a software installer is deleted from Fleet.`, `This activity contains the following fields: +- "software_title": Name of the software. +- "software_package": Filename of the installer. +- "team_name": Name of the team to which this software was added.` + " `null " + `if it was added to no team. +- "team_id": The ID of the team to which this software was added.` + " `null` " + `if it was added to no team.`, + `{ + "software_title": "Falcon.app", + "software_package": "FalconSensor-6.44.pkg", + "team_name": "Workstations", + "team_id": 123 +} +` +} + // LogRoleChangeActivities logs activities for each role change, globally and one for each change in teams. func LogRoleChangeActivities(ctx context.Context, ds Datastore, adminUser *User, oldGlobalRole *string, oldTeamRoles []UserTeam, user *User) error { if user.GlobalRole != nil && (oldGlobalRole == nil || *oldGlobalRole != *user.GlobalRole) { diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index 6ee0dcf87e..f6832aa419 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -68,7 +68,7 @@ type SoftwareInstaller struct { // no team. TeamID *uint `json:"team_id" db:"team_id"` // TitleID is the id of the software title associated with the software installer. - TitleID uint `json:"-" db:"title_id"` + TitleID *uint `json:"-" db:"title_id"` // Name is the name of the software package. Name string `json:"name" db:"filename"` // Version is the version of the software package. @@ -89,6 +89,8 @@ type SoftwareInstaller struct { PostInstallScriptContentID *uint `json:"-" db:"post_install_script_content_id"` // StorageID is the unique identifier for the software package in the software installer store. StorageID string `json:"-" db:"storage_id"` + // SoftwareTitle is the title of the software pointed installed by this installer. + SoftwareTitle string `json:"-" db:"software_title"` } // AuthzType implements authz.AuthzTyper. diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index c3f064d88f..0a884fdb05 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -11675,5 +11675,4 @@ func (s *integrationTestSuite) TestAutofillPolicies() { s.Do("PATCH", "/api/latest/fleet/config", appConfigSpec, http.StatusOK) resp = s.Do("POST", "/api/latest/fleet/autofill/policy", req, http.StatusBadRequest) assertBodyContains(t, resp, "AI features are disabled") - } diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 078749e83c..bc0f253021 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -8549,7 +8549,7 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadDownloadAndDelete() require.Equal(t, payload.StorageID, meta.StorageID) require.Equal(t, payload.Filename, meta.Name) require.Equal(t, payload.Version, meta.Version) - require.Equal(t, checkSoftwareTitle(t, payload.Title, "deb_packages"), meta.TitleID) + require.Equal(t, checkSoftwareTitle(t, payload.Title, "deb_packages"), *meta.TitleID) require.NotZero(t, meta.UploadedAt) return meta.InstallerID @@ -8570,6 +8570,9 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadDownloadAndDelete() s.uploadSoftwareInstaller(payload, http.StatusOK, "") + // check activity + s.lastActivityOfTypeMatches(fleet.ActivityTypeAddedSoftware{}.ActivityName(), `{"software_title": "ruby", "software_package": "ruby.deb", "team_name": null, "team_id": null}`, 0) + // check the software installer installerID := checkSoftwareInstaller(t, payload) @@ -8590,6 +8593,9 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadDownloadAndDelete() // delete the installer s.Do("DELETE", fmt.Sprintf("/api/latest/fleet/software/package/%d", installerID), nil, http.StatusNoContent) + + // check activity + s.lastActivityOfTypeMatches(fleet.ActivityTypeDeletedSoftware{}.ActivityName(), `{"software_title": "ruby", "software_package": "ruby.deb", "team_name": null, "team_id": null}`, 0) }) t.Run("create team software installer", func(t *testing.T) { @@ -8617,6 +8623,9 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadDownloadAndDelete() // check the software installer installerID := checkSoftwareInstaller(t, payload) + // check activity + s.lastActivityOfTypeMatches(fleet.ActivityTypeAddedSoftware{}.ActivityName(), fmt.Sprintf(`{"software_title": "ruby", "software_package": "ruby.deb", "team_name": "%s", "team_id": %d}`, createTeamResp.Team.Name, createTeamResp.Team.ID), 0) + // upload again fails s.uploadSoftwareInstaller(payload, http.StatusConflict, "already exists") @@ -8635,6 +8644,9 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadDownloadAndDelete() // delete the installer s.Do("DELETE", fmt.Sprintf("/api/latest/fleet/software/package/%d", installerID), nil, http.StatusNoContent) + + // check activity + s.lastActivityOfTypeMatches(fleet.ActivityTypeDeletedSoftware{}.ActivityName(), fmt.Sprintf(`{"software_title": "ruby", "software_package": "ruby.deb", "team_name": "%s", "team_id": %d}`, createTeamResp.Team.Name, createTeamResp.Team.ID), 0) }) } diff --git a/server/service/software_installers_test.go b/server/service/software_installers_test.go index 6e644c7a1f..991e8e89e6 100644 --- a/server/service/software_installers_test.go +++ b/server/service/software_installers_test.go @@ -67,6 +67,18 @@ func TestSoftwareInstallersAuth(t *testing.T) { return nil } + ds.NewActivityFunc = func(ctx context.Context, user *fleet.User, activity fleet.ActivityDetails) error { + return nil + } + + ds.TeamFunc = func(ctx context.Context, tid uint) (*fleet.Team, error) { + if tt.teamID != nil { + return &fleet.Team{ID: *tt.teamID}, nil + } + + return nil, nil + } + _, err := svc.DownloadSoftwareInstaller(ctx, 1) checkAuthErr(t, tt.shouldFailRead, err) From 83671662786475b340771e3eaa2c6386e267fcc2 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 8 May 2024 12:41:57 -0400 Subject: [PATCH 24/56] Add uninstalled but available software installers to the "List software titles" API (#18842) --- ...ailable-installers-to-list-software-titles | 1 + orbit/pkg/scripts/exec_nonwindows_test.go | 7 + server/datastore/mysql/software_titles.go | 28 +- .../datastore/mysql/software_titles_test.go | 279 ++++++++++++++---- 4 files changed, 254 insertions(+), 61 deletions(-) create mode 100644 changes/18831-add-available-installers-to-list-software-titles diff --git a/changes/18831-add-available-installers-to-list-software-titles b/changes/18831-add-available-installers-to-list-software-titles new file mode 100644 index 0000000000..8b01544862 --- /dev/null +++ b/changes/18831-add-available-installers-to-list-software-titles @@ -0,0 +1 @@ +* Added the uninstalled but available software installers to the response payload of the "List software titles" endpoint (`GET /software/titles`). diff --git a/orbit/pkg/scripts/exec_nonwindows_test.go b/orbit/pkg/scripts/exec_nonwindows_test.go index 52a5c069c0..4965b08838 100644 --- a/orbit/pkg/scripts/exec_nonwindows_test.go +++ b/orbit/pkg/scripts/exec_nonwindows_test.go @@ -5,6 +5,7 @@ package scripts import ( "context" "os" + "os/exec" "path/filepath" "runtime" "strings" @@ -59,6 +60,12 @@ func TestExecCmdNonWindows(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { + if strings.HasPrefix(tc.contents, "#!"+zshPath) { + // skip if zsh is not installed + if _, err := exec.LookPath(zshPath); err != nil { + t.Skipf("zsh not installed: %s", err) + } + } scriptPath := strings.ReplaceAll(tc.name, " ", "_") + ".sh" scriptPath = filepath.Join(tmpDir, scriptPath) err := os.WriteFile(scriptPath, []byte(tc.contents), os.ModePerm) diff --git a/server/datastore/mysql/software_titles.go b/server/datastore/mysql/software_titles.go index ea52931d7d..0140dc0ffc 100644 --- a/server/datastore/mysql/software_titles.go +++ b/server/datastore/mysql/software_titles.go @@ -191,15 +191,18 @@ SELECT st.name, st.source, st.browser, - MAX(sthc.hosts_count) as hosts_count, - MAX(sthc.updated_at) as counts_updated_at + MAX(COALESCE(sthc.hosts_count, 0)) as hosts_count, + MAX(COALESCE(sthc.updated_at, date('0001-01-01 00:00:00'))) as counts_updated_at FROM software_titles st -JOIN software_titles_host_counts sthc ON sthc.software_title_id = st.id +LEFT JOIN software_titles_host_counts sthc ON sthc.software_title_id = st.id AND sthc.team_id = ? -- placeholder for JOIN on software/software_cve %s -- placeholder for optional extra WHERE filter -WHERE sthc.team_id = ? %s -AND sthc.hosts_count > 0 +WHERE %s +AND ( + sthc.hosts_count > 0 OR + EXISTS (SELECT 1 FROM software_installers si WHERE si.title_id = st.id AND si.global_or_team_id = ?) +) GROUP BY st.id` cveJoinType := "LEFT" @@ -207,27 +210,34 @@ GROUP BY st.id` cveJoinType = "INNER" } + var globalOrTeamID uint args := []any{0} if opt.TeamID != nil { args[0] = *opt.TeamID + globalOrTeamID = *opt.TeamID } - additionalWhere := "" + additionalWhere := "TRUE" match := opt.ListOptions.MatchQuery softwareJoin := "" if match != "" || opt.VulnerableOnly { + // if we do a match but not vulnerable only, we want a LEFT JOIN on + // software because software installers may not have entries in software + // for their software title. If we do want vulnerable only, then we have to + // INNER JOIN because a CVE implies a specific software version. softwareJoin = fmt.Sprintf(` - JOIN software s ON s.title_id = st.id + %s JOIN software s ON s.title_id = st.id -- placeholder for changing the JOIN type to filter vulnerable software - %s JOIN software_cve scve ON s.id = scve.software_id + %[1]s JOIN software_cve scve ON s.id = scve.software_id `, cveJoinType) } if match != "" { - additionalWhere += " AND (st.name LIKE ? OR scve.cve LIKE ?)" + additionalWhere = " (st.name LIKE ? OR scve.cve LIKE ?)" match = likePattern(match) args = append(args, match, match) } + args = append(args, globalOrTeamID) stmt = fmt.Sprintf(stmt, softwareJoin, additionalWhere) return stmt, args diff --git a/server/datastore/mysql/software_titles_test.go b/server/datastore/mysql/software_titles_test.go index cf09e3326d..7fdd4cf5a5 100644 --- a/server/datastore/mysql/software_titles_test.go +++ b/server/datastore/mysql/software_titles_test.go @@ -3,11 +3,12 @@ package mysql import ( "context" "database/sql" - "github.com/stretchr/testify/assert" "sort" "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/ptr" "github.com/fleetdm/fleet/v4/server/test" @@ -24,6 +25,7 @@ func TestSoftwareTitles(t *testing.T) { {"SyncHostsSoftwareTitles", testSoftwareSyncHostsSoftwareTitles}, {"OrderSoftwareTitles", testOrderSoftwareTitles}, {"TeamFilterSoftwareTitles", testTeamFilterSoftwareTitles}, + {"ListSoftwareTitlesInstallersOnly", testListSoftwareTitlesInstallersOnly}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -70,8 +72,8 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { require.NoError(t, err) _, err = ds.UpdateHostSoftware(ctx, host2.ID, software2) require.NoError(t, err) - require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now())) + require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) globalOpts := fleet.SoftwareTitleListOptions{ListOptions: fleet.ListOptions{OrderKey: "hosts_count", OrderDirection: fleet.OrderDescending}} @@ -91,8 +93,8 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { } _, err = ds.UpdateHostSoftware(ctx, host2.ID, software2) require.NoError(t, err) - require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now())) + require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) globalCounts = listSoftwareTitlesCheckCount(t, ds, 1, 1, globalOpts, false) @@ -157,8 +159,8 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { checkTableTotalCount(1) // after a call to Calculate, the global counts are updated and the team counts appear - require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now())) + require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) globalCounts = listSoftwareTitlesCheckCount(t, ds, 2, 2, globalOpts, false) @@ -195,8 +197,8 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { _, err = ds.UpdateHostSoftware(ctx, host4.ID, software4) require.NoError(t, err) - require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now())) + require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) globalCounts = listSoftwareTitlesCheckCount(t, ds, 1, 1, globalOpts, false) @@ -223,8 +225,8 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { software4 = []fleet.Software{} _, err = ds.UpdateHostSoftware(ctx, host4.ID, software4) require.NoError(t, err) - require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now())) + require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) listSoftwareTitlesCheckCount(t, ds, 0, 0, team2Opts, false) @@ -232,8 +234,8 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { require.NoError(t, ds.DeleteTeam(ctx, team2.ID)) // this call will remove team2 from the software host counts table - require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now())) + require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) globalCounts = listSoftwareTitlesCheckCount(t, ds, 1, 1, globalOpts, false) @@ -284,8 +286,29 @@ func testOrderSoftwareTitles(t *testing.T, ds *Datastore) { require.NoError(t, err) _, err = ds.UpdateHostSoftware(ctx, host3.ID, software3) require.NoError(t, err) - require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) + + // create a software installer not installed on any host + installer1, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + Title: "installer1", + Source: "apps", + InstallScript: "echo", + Filename: "installer1.pkg", + }) + require.NoError(t, err) + require.NotZero(t, installer1) + // create a software installer with an install request on host1 + installer2, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + Title: "installer2", + Source: "apps", + InstallScript: "echo", + Filename: "installer2.pkg", + }) + require.NoError(t, err) + err = ds.InsertSoftwareInstallRequest(ctx, host1.ID, installer2) + require.NoError(t, err) + require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now())) + require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) // primary sort is "hosts_count DESC", followed by "name ASC, source ASC, browser ASC" @@ -294,7 +317,7 @@ func testOrderSoftwareTitles(t *testing.T, ds *Datastore) { OrderDirection: fleet.OrderDescending, }}, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}}) require.NoError(t, err) - require.Len(t, titles, 7) + require.Len(t, titles, 9) require.Equal(t, "bar", titles[0].Name) require.Equal(t, "deb_packages", titles[0].Source) require.Equal(t, "foo", titles[1].Name) @@ -311,6 +334,10 @@ func testOrderSoftwareTitles(t *testing.T, ds *Datastore) { require.Equal(t, "edge", titles[5].Browser) require.Equal(t, "foo", titles[6].Name) require.Equal(t, "rpm_packages", titles[6].Source) + require.Equal(t, "installer1", titles[7].Name) + require.Equal(t, "apps", titles[7].Source) + require.Equal(t, "installer2", titles[8].Name) + require.Equal(t, "apps", titles[8].Source) // primary sort is "hosts_count ASC", followed by "name ASC, source ASC, browser ASC" titles, _, _, err = ds.ListSoftwareTitles(ctx, fleet.SoftwareTitleListOptions{ListOptions: fleet.ListOptions{ @@ -318,23 +345,27 @@ func testOrderSoftwareTitles(t *testing.T, ds *Datastore) { OrderDirection: fleet.OrderAscending, }}, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}}) require.NoError(t, err) - require.Len(t, titles, 7) - require.Equal(t, "bar", titles[0].Name) + require.Len(t, titles, 9) + require.Equal(t, "installer1", titles[0].Name) require.Equal(t, "apps", titles[0].Source) - require.Equal(t, "baz", titles[1].Name) - require.Equal(t, "chrome_extensions", titles[1].Source) - require.Equal(t, "chrome", titles[1].Browser) - require.Equal(t, "baz", titles[2].Name) - require.Equal(t, "chrome_extensions", titles[2].Source) - require.Equal(t, "edge", titles[2].Browser) - require.Equal(t, "foo", titles[3].Name) - require.Equal(t, "rpm_packages", titles[3].Source) - require.Equal(t, "bar", titles[4].Name) - require.Equal(t, "deb_packages", titles[4].Source) + require.Equal(t, "installer2", titles[1].Name) + require.Equal(t, "apps", titles[1].Source) + require.Equal(t, "bar", titles[2].Name) + require.Equal(t, "apps", titles[2].Source) + require.Equal(t, "baz", titles[3].Name) + require.Equal(t, "chrome_extensions", titles[3].Source) + require.Equal(t, "chrome", titles[3].Browser) + require.Equal(t, "baz", titles[4].Name) + require.Equal(t, "chrome_extensions", titles[4].Source) + require.Equal(t, "edge", titles[4].Browser) require.Equal(t, "foo", titles[5].Name) - require.Equal(t, "chrome_extensions", titles[5].Source) - require.Equal(t, "foo", titles[6].Name) + require.Equal(t, "rpm_packages", titles[5].Source) + require.Equal(t, "bar", titles[6].Name) require.Equal(t, "deb_packages", titles[6].Source) + require.Equal(t, "foo", titles[7].Name) + require.Equal(t, "chrome_extensions", titles[7].Source) + require.Equal(t, "foo", titles[8].Name) + require.Equal(t, "deb_packages", titles[8].Source) // primary sort is "name ASC", followed by "host_count DESC, source ASC, browser ASC" titles, _, _, err = ds.ListSoftwareTitles(ctx, fleet.SoftwareTitleListOptions{ListOptions: fleet.ListOptions{ @@ -342,7 +373,7 @@ func testOrderSoftwareTitles(t *testing.T, ds *Datastore) { OrderDirection: fleet.OrderAscending, }}, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}}) require.NoError(t, err) - require.Len(t, titles, 7) + require.Len(t, titles, 9) require.Equal(t, "bar", titles[0].Name) require.Equal(t, "deb_packages", titles[0].Source) require.Equal(t, "bar", titles[1].Name) @@ -359,6 +390,10 @@ func testOrderSoftwareTitles(t *testing.T, ds *Datastore) { require.Equal(t, "deb_packages", titles[5].Source) require.Equal(t, "foo", titles[6].Name) require.Equal(t, "rpm_packages", titles[6].Source) + require.Equal(t, "installer1", titles[7].Name) + require.Equal(t, "apps", titles[7].Source) + require.Equal(t, "installer2", titles[8].Name) + require.Equal(t, "apps", titles[8].Source) // primary sort is "name DESC", followed by "host_count DESC, source ASC, browser ASC" titles, _, _, err = ds.ListSoftwareTitles(ctx, fleet.SoftwareTitleListOptions{ListOptions: fleet.ListOptions{ @@ -366,23 +401,59 @@ func testOrderSoftwareTitles(t *testing.T, ds *Datastore) { OrderDirection: fleet.OrderDescending, }}, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}}) require.NoError(t, err) - require.Len(t, titles, 7) - require.Equal(t, "foo", titles[0].Name) - require.Equal(t, "chrome_extensions", titles[0].Source) - require.Equal(t, "foo", titles[1].Name) - require.Equal(t, "deb_packages", titles[1].Source) + require.Len(t, titles, 9) + require.Equal(t, "installer2", titles[0].Name) + require.Equal(t, "apps", titles[0].Source) + require.Equal(t, "installer1", titles[1].Name) + require.Equal(t, "apps", titles[1].Source) require.Equal(t, "foo", titles[2].Name) - require.Equal(t, "rpm_packages", titles[2].Source) - require.Equal(t, "baz", titles[3].Name) - require.Equal(t, "chrome_extensions", titles[3].Source) - require.Equal(t, "chrome", titles[3].Browser) - require.Equal(t, "baz", titles[4].Name) - require.Equal(t, "chrome_extensions", titles[4].Source) - require.Equal(t, "edge", titles[4].Browser) - require.Equal(t, "bar", titles[5].Name) - require.Equal(t, "deb_packages", titles[5].Source) - require.Equal(t, "bar", titles[6].Name) - require.Equal(t, "apps", titles[6].Source) + require.Equal(t, "chrome_extensions", titles[2].Source) + require.Equal(t, "foo", titles[3].Name) + require.Equal(t, "deb_packages", titles[3].Source) + require.Equal(t, "foo", titles[4].Name) + require.Equal(t, "rpm_packages", titles[4].Source) + require.Equal(t, "baz", titles[5].Name) + require.Equal(t, "chrome_extensions", titles[5].Source) + require.Equal(t, "chrome", titles[5].Browser) + require.Equal(t, "baz", titles[6].Name) + require.Equal(t, "chrome_extensions", titles[6].Source) + require.Equal(t, "edge", titles[6].Browser) + require.Equal(t, "bar", titles[7].Name) + require.Equal(t, "deb_packages", titles[7].Source) + require.Equal(t, "bar", titles[8].Name) + require.Equal(t, "apps", titles[8].Source) + + // using a match query + titles, _, _, err = ds.ListSoftwareTitles(ctx, fleet.SoftwareTitleListOptions{ListOptions: fleet.ListOptions{ + OrderKey: "name", + OrderDirection: fleet.OrderDescending, + MatchQuery: "ba", + }}, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}}) + require.NoError(t, err) + require.Len(t, titles, 4) + require.Equal(t, "baz", titles[0].Name) + require.Equal(t, "chrome_extensions", titles[0].Source) + require.Equal(t, "chrome", titles[0].Browser) + require.Equal(t, "baz", titles[1].Name) + require.Equal(t, "chrome_extensions", titles[1].Source) + require.Equal(t, "edge", titles[1].Browser) + require.Equal(t, "bar", titles[2].Name) + require.Equal(t, "deb_packages", titles[2].Source) + require.Equal(t, "bar", titles[3].Name) + require.Equal(t, "apps", titles[3].Source) + + // using another (installer-only) match query + titles, _, _, err = ds.ListSoftwareTitles(ctx, fleet.SoftwareTitleListOptions{ListOptions: fleet.ListOptions{ + OrderKey: "name", + OrderDirection: fleet.OrderDescending, + MatchQuery: "insta", + }}, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}}) + require.NoError(t, err) + require.Len(t, titles, 2) + require.Equal(t, "installer2", titles[0].Name) + require.Equal(t, "apps", titles[0].Source) + require.Equal(t, "installer1", titles[1].Name) + require.Equal(t, "apps", titles[1].Source) } func listSoftwareTitlesCheckCount(t *testing.T, ds *Datastore, expectedListCount int, expectedFullCount int, opts fleet.SoftwareTitleListOptions, returnSorted bool) []fleet.SoftwareTitle { @@ -407,11 +478,11 @@ func testTeamFilterSoftwareTitles(t *testing.T, ds *Datastore) { host2 := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now()) require.NoError(t, ds.AddHostsToTeam(ctx, &team2.ID, []uint{host2.ID})) - user1, err := ds.NewUser(ctx, &fleet.User{Name: "user1", Password: []byte("test"), Email: "test1@email.com", GlobalRole: ptr.String(fleet.RoleAdmin)}) + userGlobalAdmin, err := ds.NewUser(ctx, &fleet.User{Name: "user1", Password: []byte("test"), Email: "test1@email.com", GlobalRole: ptr.String(fleet.RoleAdmin)}) require.NoError(t, err) - user2, err := ds.NewUser(ctx, &fleet.User{Name: "user2", Password: []byte("test"), Email: "test2@email.com", Teams: []fleet.UserTeam{{Team: fleet.Team{ID: team1.ID}, Role: fleet.RoleAdmin}}}) + userTeam1Admin, err := ds.NewUser(ctx, &fleet.User{Name: "user2", Password: []byte("test"), Email: "test2@email.com", Teams: []fleet.UserTeam{{Team: fleet.Team{ID: team1.ID}, Role: fleet.RoleAdmin}}}) require.NoError(t, err) - user3, err := ds.NewUser(ctx, &fleet.User{Name: "user3", Password: []byte("test"), Email: "test3@email.com", Teams: []fleet.UserTeam{{Team: fleet.Team{ID: team2.ID}, Role: fleet.RoleAdmin}}}) + userTeam2Admin, err := ds.NewUser(ctx, &fleet.User{Name: "user3", Password: []byte("test"), Email: "test3@email.com", Teams: []fleet.UserTeam{{Team: fleet.Team{ID: team2.ID}, Role: fleet.RoleAdmin}}}) require.NoError(t, err) software1 := []fleet.Software{ @@ -427,19 +498,47 @@ func testTeamFilterSoftwareTitles(t *testing.T, ds *Datastore) { _, err = ds.UpdateHostSoftware(ctx, host2.ID, software2) require.NoError(t, err) - require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) + // create a software installer for team1 + installer1, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + Title: "installer1", + Source: "apps", + InstallScript: "echo", + Filename: "installer1.pkg", + TeamID: &team1.ID, + }) + require.NoError(t, err) + require.NotZero(t, installer1) + // create a software installer for team2 + installer2, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + Title: "installer2", + Source: "apps", + InstallScript: "echo", + Filename: "installer2.pkg", + TeamID: &team2.ID, + }) + require.NoError(t, err) + require.NotZero(t, installer2) + require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now())) + require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) - // Testing the global user - globalTeamFilter := fleet.TeamFilter{User: user1, IncludeObserver: true} + // Testing the global user (for no team) + globalTeamFilter := fleet.TeamFilter{User: userGlobalAdmin, IncludeObserver: true} titles, count, _, err := ds.ListSoftwareTitles( context.Background(), fleet.SoftwareTitleListOptions{ListOptions: fleet.ListOptions{}}, globalTeamFilter, ) sortTitlesByName(titles) + // software installers are associated with a team, so they don't show up in + // this request for no team, but other titles do because software titles are + // not associated with a team. require.NoError(t, err) require.Len(t, titles, 2) require.Equal(t, 2, count) + require.Equal(t, "bar", titles[0].Name) + require.Equal(t, "deb_packages", titles[0].Source) + require.Equal(t, "foo", titles[1].Name) + require.Equal(t, "chrome_extensions", titles[1].Source) require.Equal(t, uint(1), titles[0].VersionsCount) assert.Equal(t, uint(1), titles[0].HostsCount) require.Equal(t, uint(2), titles[1].VersionsCount) @@ -465,14 +564,20 @@ func testTeamFilterSoftwareTitles(t *testing.T, ds *Datastore) { assert.Equal(t, "0.0.3", title.Versions[0].Version) // Testing the team 1 user - team1TeamFilter := fleet.TeamFilter{User: user2, IncludeObserver: true} + team1TeamFilter := fleet.TeamFilter{User: userTeam1Admin, IncludeObserver: true} titles, count, _, err = ds.ListSoftwareTitles( context.Background(), fleet.SoftwareTitleListOptions{ListOptions: fleet.ListOptions{}, TeamID: &team1.ID}, team1TeamFilter, ) + // installer1 is associated with team 1 require.NoError(t, err) - require.Len(t, titles, 1) - require.Equal(t, 1, count) + require.Len(t, titles, 2) + require.Equal(t, 2, count) + require.Equal(t, "foo", titles[0].Name) + require.Equal(t, "chrome_extensions", titles[0].Source) + require.Equal(t, "installer1", titles[1].Name) + require.Equal(t, "apps", titles[1].Source) require.Equal(t, uint(1), titles[0].VersionsCount) + require.Equal(t, uint(0), titles[1].VersionsCount) // Testing with team filter -- this team does contain this software title title, err = ds.SoftwareTitleByID(context.Background(), titles[0].ID, &team1.ID, team1TeamFilter) @@ -483,16 +588,86 @@ func testTeamFilterSoftwareTitles(t *testing.T, ds *Datastore) { // Testing the team 2 user titles, count, _, err = ds.ListSoftwareTitles(context.Background(), fleet.SoftwareTitleListOptions{ListOptions: fleet.ListOptions{}, TeamID: &team2.ID}, fleet.TeamFilter{ - User: user3, + User: userTeam2Admin, IncludeObserver: true, }) + // installer2 is associated with team 2 require.NoError(t, err) - require.Len(t, titles, 2) - require.Equal(t, 2, count) + require.Len(t, titles, 3) + require.Equal(t, 3, count) + require.Equal(t, "bar", titles[0].Name) + require.Equal(t, "deb_packages", titles[0].Source) + require.Equal(t, "foo", titles[1].Name) + require.Equal(t, "chrome_extensions", titles[1].Source) + require.Equal(t, "installer2", titles[2].Name) + require.Equal(t, "apps", titles[2].Source) require.Equal(t, uint(1), titles[0].VersionsCount) require.Equal(t, uint(1), titles[1].VersionsCount) + require.Equal(t, uint(0), titles[2].VersionsCount) } func sortTitlesByName(titles []fleet.SoftwareTitle) { sort.Slice(titles, func(i, j int) bool { return titles[i].Name < titles[j].Name }) } + +func testListSoftwareTitlesInstallersOnly(t *testing.T, ds *Datastore) { + ctx := context.Background() + + // create a couple software installers not installed on any host + installer1, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + Title: "installer1", + Source: "apps", + InstallScript: "echo", + Filename: "installer1.pkg", + }) + require.NoError(t, err) + require.NotZero(t, installer1) + installer2, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + Title: "installer2", + Source: "apps", + InstallScript: "echo", + Filename: "installer2.pkg", + }) + require.NoError(t, err) + require.NotZero(t, installer2) + + titles, counts, _, err := ds.ListSoftwareTitles(ctx, fleet.SoftwareTitleListOptions{ListOptions: fleet.ListOptions{ + OrderKey: "name", + OrderDirection: fleet.OrderAscending, + }}, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}}) + require.NoError(t, err) + require.EqualValues(t, 2, counts) + require.Len(t, titles, 2) + require.Equal(t, "installer1", titles[0].Name) + require.Equal(t, "apps", titles[0].Source) + require.Equal(t, "installer2", titles[1].Name) + require.Equal(t, "apps", titles[1].Source) + require.True(t, titles[0].CountsUpdatedAt.IsZero()) + require.True(t, titles[1].CountsUpdatedAt.IsZero()) + + require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now())) + require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) + require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) + + titles, counts, _, err = ds.ListSoftwareTitles(ctx, fleet.SoftwareTitleListOptions{ListOptions: fleet.ListOptions{ + OrderKey: "name", + OrderDirection: fleet.OrderAscending, + MatchQuery: "installer1", + }}, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}}) + require.NoError(t, err) + require.EqualValues(t, 1, counts) + require.Len(t, titles, 1) + require.Equal(t, "installer1", titles[0].Name) + require.Equal(t, "apps", titles[0].Source) + require.True(t, titles[0].CountsUpdatedAt.IsZero()) + + // vulnerable only returns nothing + titles, counts, _, err = ds.ListSoftwareTitles(ctx, fleet.SoftwareTitleListOptions{ListOptions: fleet.ListOptions{ + OrderKey: "name", + OrderDirection: fleet.OrderAscending, + MatchQuery: "installer1", + }, VulnerableOnly: true}, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}}) + require.NoError(t, err) + require.EqualValues(t, 0, counts) + require.Len(t, titles, 0) +} From 2a4b00b349911f2bd949e16e0ac589e9476beba7 Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Wed, 8 May 2024 15:52:35 -0500 Subject: [PATCH 25/56] Add software installer details to get software title API response and add software install status filter to list hosts API (#18748) --- cmd/fleetctl/get_test.go | 8 +- server/datastore/mysql/hosts.go | 61 +++++--- server/datastore/mysql/labels.go | 52 +++++-- server/datastore/mysql/software_installers.go | 131 ++++++++++++++++++ .../mysql/software_installers_test.go | 24 ++++ server/datastore/mysql/software_test.go | 14 +- server/datastore/mysql/software_titles.go | 19 ++- server/fleet/datastore.go | 10 +- server/fleet/hosts.go | 4 + server/fleet/software.go | 4 +- server/fleet/software_installer.go | 24 +++- server/mock/datastore_mock.go | 24 ++++ server/service/hosts_test.go | 3 + server/service/integration_mdm_test.go | 116 ++++++++++++++-- server/service/software_titles.go | 24 +++- server/service/transport.go | 24 ++++ 16 files changed, 483 insertions(+), 59 deletions(-) diff --git a/cmd/fleetctl/get_test.go b/cmd/fleetctl/get_test.go index 9c4781d301..4b64f3c169 100644 --- a/cmd/fleetctl/get_test.go +++ b/cmd/fleetctl/get_test.go @@ -677,6 +677,7 @@ spec: - hosts_count: 2 id: 0 name: foo + software_package: null source: chrome_extensions versions: - id: 0 @@ -696,6 +697,7 @@ spec: - hosts_count: 0 id: 0 name: bar + software_package: null source: deb_packages versions: - id: 0 @@ -738,7 +740,8 @@ spec: "cve-123-456-003" ] } - ] + ], + "software_package": null }, { "id": 0, @@ -752,7 +755,8 @@ spec: "version": "0.0.3", "vulnerabilities": null } - ] + ], + "software_package": null } ] } diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index 70cd850289..721750345a 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -975,9 +975,12 @@ func (ds *Datastore) ListHosts(ctx context.Context, filter fleet.TeamFilter, opt // TODO(Sarah): Do we need to reconcile mutually exclusive filters? func (ds *Datastore) applyHostFilters( - ctx context.Context, opt fleet.HostListOptions, sqlStmt string, filter fleet.TeamFilter, params []interface{}, + ctx context.Context, opt fleet.HostListOptions, sqlStmt string, filter fleet.TeamFilter, selectParams []interface{}, leftJoinFailingPolicies bool, ) (string, []interface{}, error) { + // prior to returning, params will be appended in the following order: selectParams, joinParams, whereParams + var whereParams, joinParams []interface{} + opt.OrderKey = defaultHostColumnTableAlias(opt.OrderKey) deviceMappingJoin := fmt.Sprintf(`LEFT JOIN ( @@ -999,6 +1002,7 @@ func (ds *Datastore) applyHostFilters( policyMembershipJoin = "LEFT " + policyMembershipJoin } + softwareStatusJoin := "" softwareFilter := "TRUE" var softwareIDFilter *uint if opt.SoftwareVersionIDFilter != nil { @@ -1008,12 +1012,26 @@ func (ds *Datastore) applyHostFilters( } if softwareIDFilter != nil { softwareFilter = "EXISTS (SELECT 1 FROM host_software hs WHERE hs.host_id = h.id AND hs.software_id = ?)" - params = append(params, *softwareIDFilter) + whereParams = append(whereParams, *softwareIDFilter) } else if opt.SoftwareTitleIDFilter != nil { // software (version) ID filter is mutually exclusive with software title ID // so we're reusing the same filter to avoid adding unnecessary conditions. - softwareFilter = "EXISTS (SELECT 1 FROM host_software hs INNER JOIN software sw ON hs.software_id = sw.id WHERE hs.host_id = h.id AND sw.title_id = ?)" - params = append(params, *opt.SoftwareTitleIDFilter) + if opt.SoftwareStatusFilter != nil { + // get the installer id + meta, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, opt.TeamFilter, *opt.SoftwareTitleIDFilter) + if err != nil { + return "", nil, ctxerr.Wrap(ctx, err, "get software installer metadata by team and title id") + } + installerJoin, installerParams, err := ds.softwareInstallerJoin(meta.InstallerID, *opt.SoftwareStatusFilter) + if err != nil { + return "", nil, ctxerr.Wrap(ctx, err, "software installer join") + } + softwareStatusJoin = installerJoin + joinParams = append(joinParams, installerParams...) + } else { + softwareFilter = "EXISTS (SELECT 1 FROM host_software hs INNER JOIN software sw ON hs.software_id = sw.id WHERE hs.host_id = h.id AND sw.title_id = ?)" + whereParams = append(whereParams, *opt.SoftwareTitleIDFilter) + } } failingPoliciesJoin := "" @@ -1034,7 +1052,7 @@ func (ds *Datastore) applyHostFilters( if opt.MunkiIssueIDFilter != nil { munkiJoin = ` JOIN host_munki_issues hmi ON h.id = hmi.host_id ` munkiFilter = "hmi.munki_issue_id = ?" - params = append(params, opt.MunkiIssueIDFilter) + whereParams = append(whereParams, opt.MunkiIssueIDFilter) } displayNameJoin := "" @@ -1048,7 +1066,7 @@ func (ds *Datastore) applyHostFilters( lowDiskSpaceFilter := "TRUE" if opt.LowDiskSpaceFilter != nil { lowDiskSpaceFilter = `hd.gigs_disk_space_available < ?` - params = append(params, *opt.LowDiskSpaceFilter) + whereParams = append(whereParams, *opt.LowDiskSpaceFilter) } sqlStmt += fmt.Sprintf( @@ -1064,6 +1082,7 @@ func (ds *Datastore) applyHostFilters( %s %s %s + %s WHERE TRUE AND %s AND %s AND %s AND %s `, @@ -1071,6 +1090,7 @@ func (ds *Datastore) applyHostFilters( hostMDMJoin, deviceMappingJoin, policyMembershipJoin, + softwareStatusJoin, failingPoliciesJoin, operatingSystemJoin, munkiJoin, @@ -1084,16 +1104,16 @@ func (ds *Datastore) applyHostFilters( ) now := ds.clock.Now() - sqlStmt, params = filterHostsByStatus(now, sqlStmt, opt, params) - sqlStmt, params = filterHostsByTeam(sqlStmt, opt, params) - sqlStmt, params = filterHostsByPolicy(sqlStmt, opt, params) - sqlStmt, params = filterHostsByMDM(sqlStmt, opt, params) + sqlStmt, whereParams = filterHostsByStatus(now, sqlStmt, opt, whereParams) + sqlStmt, whereParams = filterHostsByTeam(sqlStmt, opt, whereParams) + sqlStmt, whereParams = filterHostsByPolicy(sqlStmt, opt, whereParams) + sqlStmt, whereParams = filterHostsByMDM(sqlStmt, opt, whereParams) var err error - sqlStmt, params, err = filterHostsByMacOSSettingsStatus(sqlStmt, opt, params) + sqlStmt, whereParams, err = filterHostsByMacOSSettingsStatus(sqlStmt, opt, whereParams) if err != nil { return "", nil, ctxerr.Wrap(ctx, err, "building query to filter macOS settings status") } - sqlStmt, params = filterHostsByMacOSDiskEncryptionStatus(sqlStmt, opt, params) + sqlStmt, whereParams = filterHostsByMacOSDiskEncryptionStatus(sqlStmt, opt, whereParams) if enableDiskEncryption, err := ds.getConfigEnableDiskEncryption(ctx, opt.TeamFilter); err != nil { if errors.Is(err, sql.ErrNoRows) { return "", nil, ctxerr.Wrap( @@ -1105,19 +1125,22 @@ func (ds *Datastore) applyHostFilters( } return "", nil, err } else if opt.OSSettingsFilter.IsValid() { - sqlStmt, params, err = ds.filterHostsByOSSettingsStatus(sqlStmt, opt, params, enableDiskEncryption) + sqlStmt, whereParams, err = ds.filterHostsByOSSettingsStatus(sqlStmt, opt, whereParams, enableDiskEncryption) if err != nil { return "", nil, err } } else if opt.OSSettingsDiskEncryptionFilter.IsValid() { - sqlStmt, params = ds.filterHostsByOSSettingsDiskEncryptionStatus(sqlStmt, opt, params, enableDiskEncryption) + sqlStmt, whereParams = ds.filterHostsByOSSettingsDiskEncryptionStatus(sqlStmt, opt, whereParams, enableDiskEncryption) } - sqlStmt, params = filterHostsByMDMBootstrapPackageStatus(sqlStmt, opt, params) - sqlStmt, params = filterHostsByOS(sqlStmt, opt, params) - sqlStmt, params = filterHostsByVulnerability(sqlStmt, opt, params) - sqlStmt, params, _ = hostSearchLike(sqlStmt, params, opt.MatchQuery, append(hostSearchColumns, "display_name")...) - sqlStmt, params = appendListOptionsWithCursorToSQL(sqlStmt, params, &opt.ListOptions) + sqlStmt, whereParams = filterHostsByMDMBootstrapPackageStatus(sqlStmt, opt, whereParams) + sqlStmt, whereParams = filterHostsByOS(sqlStmt, opt, whereParams) + sqlStmt, whereParams = filterHostsByVulnerability(sqlStmt, opt, whereParams) + sqlStmt, whereParams, _ = hostSearchLike(sqlStmt, whereParams, opt.MatchQuery, append(hostSearchColumns, "display_name")...) + sqlStmt, whereParams = appendListOptionsWithCursorToSQL(sqlStmt, whereParams, &opt.ListOptions) + + params := append(selectParams, joinParams...) + params = append(params, whereParams...) return sqlStmt, params, nil } diff --git a/server/datastore/mysql/labels.go b/server/datastore/mysql/labels.go index 272fd7fe12..e119ec3ec7 100644 --- a/server/datastore/mysql/labels.go +++ b/server/datastore/mysql/labels.go @@ -589,43 +589,69 @@ func (ds *Datastore) ListHostsInLabel(ctx context.Context, filter fleet.TeamFilt // NOTE: the hosts table must be aliased to `h` in the query passed to this function. func (ds *Datastore) applyHostLabelFilters(ctx context.Context, filter fleet.TeamFilter, lid uint, query string, opt fleet.HostListOptions) (string, []interface{}, error) { - params := []interface{}{lid} + // prior to returning, params will be appended in the following order: joinParams, whereParams + var whereParams, joinParams []interface{} if opt.ListOptions.OrderKey == "display_name" { query += ` JOIN host_display_names hdn ON h.id = hdn.host_id ` } + softwareStatusJoin := "" + // if opt.SoftwareVersionIDFilter != nil { + // // TODO: Do we currently support filtering by software version ID and label? + // } else if opt.SoftwareIDFilter != nil { + // // TODO: Do we currently support filtering by software version ID and label? + // } + if opt.SoftwareTitleIDFilter != nil && opt.SoftwareStatusFilter != nil { + // get the installer id + meta, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, opt.TeamFilter, *opt.SoftwareTitleIDFilter) + if err != nil { + return "", nil, ctxerr.Wrap(ctx, err, "get software installer metadata by team and title id") + } + installerJoin, installerParams, err := ds.softwareInstallerJoin(meta.InstallerID, *opt.SoftwareStatusFilter) + if err != nil { + return "", nil, ctxerr.Wrap(ctx, err, "software installer join") + } + softwareStatusJoin = installerJoin + joinParams = append(joinParams, installerParams...) + } + if softwareStatusJoin != "" { + query += softwareStatusJoin + } + query += fmt.Sprintf(` WHERE lm.label_id = ? AND %s `, ds.whereFilterHostsByTeams(filter, "h")) + whereParams = append(whereParams, lid) + if opt.LowDiskSpaceFilter != nil { query += ` AND hd.gigs_disk_space_available < ? ` - params = append(params, *opt.LowDiskSpaceFilter) + whereParams = append(whereParams, *opt.LowDiskSpaceFilter) } var err error - query, params = filterHostsByStatus(ds.clock.Now(), query, opt, params) - query, params = filterHostsByTeam(query, opt, params) - query, params = filterHostsByMDM(query, opt, params) - query, params, err = filterHostsByMacOSSettingsStatus(query, opt, params) + query, whereParams = filterHostsByStatus(ds.clock.Now(), query, opt, whereParams) + query, whereParams = filterHostsByTeam(query, opt, whereParams) + query, whereParams = filterHostsByMDM(query, opt, whereParams) + query, whereParams, err = filterHostsByMacOSSettingsStatus(query, opt, whereParams) if err != nil { return "", nil, ctxerr.Wrap(ctx, err, "building macOS settings status filter") } - query, params = filterHostsByMacOSDiskEncryptionStatus(query, opt, params) - query, params = filterHostsByMDMBootstrapPackageStatus(query, opt, params) + query, whereParams = filterHostsByMacOSDiskEncryptionStatus(query, opt, whereParams) + query, whereParams = filterHostsByMDMBootstrapPackageStatus(query, opt, whereParams) if enableDiskEncryption, err := ds.getConfigEnableDiskEncryption(ctx, opt.TeamFilter); err != nil { return "", nil, err } else if opt.OSSettingsFilter.IsValid() { - query, params, err = ds.filterHostsByOSSettingsStatus(query, opt, params, enableDiskEncryption) + query, whereParams, err = ds.filterHostsByOSSettingsStatus(query, opt, whereParams, enableDiskEncryption) if err != nil { return "", nil, err } } else if opt.OSSettingsDiskEncryptionFilter.IsValid() { - query, params = ds.filterHostsByOSSettingsDiskEncryptionStatus(query, opt, params, enableDiskEncryption) + query, whereParams = ds.filterHostsByOSSettingsDiskEncryptionStatus(query, opt, whereParams, enableDiskEncryption) } // TODO: should search columns include display_name (requires join to host_display_names)? - query, params, _ = hostSearchLike(query, params, opt.MatchQuery, hostSearchColumns...) + query, whereParams, _ = hostSearchLike(query, whereParams, opt.MatchQuery, hostSearchColumns...) - query, params = appendListOptionsWithCursorToSQL(query, params, &opt.ListOptions) - return query, params, nil + query, whereParams = appendListOptionsWithCursorToSQL(query, whereParams, &opt.ListOptions) + return query, append(joinParams, whereParams...), nil } func (ds *Datastore) CountHostsInLabel(ctx context.Context, filter fleet.TeamFilter, lid uint, opt fleet.HostListOptions) (int, error) { diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 3d1e5bf509..6a21e1801d 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -3,6 +3,7 @@ package mysql import ( "context" "database/sql" + "fmt" "github.com/fleetdm/fleet/v4/server/authz" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" @@ -180,6 +181,41 @@ WHERE return &dest, nil } +func (ds *Datastore) GetSoftwareInstallerMetadataByTeamAndTitleID(ctx context.Context, teamID *uint, titleID uint) (*fleet.SoftwareInstaller, error) { + query := ` +SELECT + id, + team_id, + title_id, + storage_id, + filename, + version, + install_script_content_id, + pre_install_query, + post_install_script_content_id, + uploaded_at +FROM + software_installers +WHERE + title_id = ? AND global_or_team_id = ?` + + var tmID uint + if teamID != nil { + tmID = *teamID + } + + var dest fleet.SoftwareInstaller + err := sqlx.GetContext(ctx, ds.reader(ctx), &dest, query, titleID, tmID) + if err != nil { + if err == sql.ErrNoRows { + return nil, ctxerr.Wrap(ctx, notFound("SoftwareInstaller"), "get software installer metadata") + } + return nil, ctxerr.Wrap(ctx, err, "get software installer metadata") + } + + return &dest, nil +} + func (ds *Datastore) DeleteSoftwareInstaller(ctx context.Context, id uint) error { res, err := ds.writer(ctx).ExecContext(ctx, `DELETE FROM software_installers WHERE id = ?`, id) if err != nil { @@ -316,6 +352,101 @@ WHERE return &dest, nil } +func tmplNamedSQLCaseHostSoftwareInstallStatus(alias string) string { + return fmt.Sprintf(` + CASE WHEN %[1]s.post_install_script_exit_code IS NOT NULL + AND %[1]s.post_install_script_exit_code = 0 THEN + :installed + WHEN %[1]s.post_install_script_exit_code IS NOT NULL + AND %[1]s.post_install_script_exit_code != 0 THEN + :failed + WHEN %[1]s.install_script_exit_code IS NOT NULL + AND %[1]s.install_script_exit_code = 0 THEN + :installed + WHEN %[1]s.install_script_exit_code IS NOT NULL + AND %[1]s.install_script_exit_code != 0 THEN + :failed + WHEN %[1]s.pre_install_query_output IS NOT NULL + AND %[1]s.pre_install_query_output = '' THEN + :failed + WHEN %[1]s.host_id IS NOT NULL THEN + :pending + ELSE + NULL -- not installed from Fleet installer + END`, alias) +} + +func (ds *Datastore) GetSummaryHostSoftwareInstalls(ctx context.Context, installerID uint) (*fleet.SoftwareInstallerStatusSummary, error) { + var dest fleet.SoftwareInstallerStatusSummary + + stmt := fmt.Sprintf(` +SELECT + COALESCE(SUM( IF(status = :pending, 1, 0)), 0) AS pending, + COALESCE(SUM( IF(status = :failed, 1, 0)), 0) AS failed, + COALESCE(SUM( IF(status = :installed, 1, 0)), 0) AS installed +FROM ( +SELECT + software_installer_id, + %s AS status +FROM + host_software_installs hsi +WHERE + software_installer_id = :installer_id + AND id IN( + SELECT + max(id) -- ensure we use only the most recently created install attempt for each host + FROM host_software_installs + WHERE + software_installer_id = :installer_id + GROUP BY + host_id)) s`, tmplNamedSQLCaseHostSoftwareInstallStatus("hsi")) + + query, args, err := sqlx.Named(stmt, map[string]interface{}{ + "installer_id": installerID, + "pending": fleet.SoftwareInstallerPending, + "failed": fleet.SoftwareInstallerFailed, + "installed": fleet.SoftwareInstallerInstalled, + }) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "get summary host software installs: named query") + } + + err = sqlx.GetContext(ctx, ds.reader(ctx), &dest, query, args...) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "get summary host software install status") + } + + return &dest, nil +} + +func (ds *Datastore) softwareInstallerJoin(installerID uint, status fleet.SoftwareInstallerStatus) (string, []interface{}, error) { + stmt := fmt.Sprintf(`JOIN ( +SELECT + host_id +FROM + host_software_installs hsi +WHERE + software_installer_id = :installer_id + AND hsi.id IN( + SELECT + max(id) -- ensure we use only the most recent install attempt for each host + FROM host_software_installs + WHERE + software_installer_id = :installer_id + GROUP BY + host_id, software_installer_id) + AND (%s) = :status) hss ON hss.host_id = h.id +`, tmplNamedSQLCaseHostSoftwareInstallStatus("hsi")) + + return sqlx.Named(stmt, map[string]interface{}{ + "status": status, + "installer_id": installerID, + "installed": fleet.SoftwareInstallerInstalled, + "failed": fleet.SoftwareInstallerFailed, + "pending": fleet.SoftwareInstallerPending, + }) +} + func (ds *Datastore) CleanupUnusedSoftwareInstallers(ctx context.Context, softwareInstallStore fleet.SoftwareInstallerStore) error { if softwareInstallStore == nil { // no-op in this case, possible if not running with a Premium license diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go index 87b2f9a140..ac0f844b7d 100644 --- a/server/datastore/mysql/software_installers_test.go +++ b/server/datastore/mysql/software_installers_test.go @@ -227,6 +227,30 @@ func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { require.NoError(t, err) err = ds.InsertSoftwareInstallRequest(ctx, host.ID, si.InstallerID) require.NoError(t, err) + + // list hosts with software install requests + userTeamFilter := fleet.TeamFilter{ + User: &fleet.User{GlobalRole: ptr.String("admin")}, + } + expectStatus := fleet.SoftwareInstallerPending + hosts, err := ds.ListHosts(ctx, userTeamFilter, fleet.HostListOptions{ + ListOptions: fleet.ListOptions{PerPage: 100}, + SoftwareTitleIDFilter: installerMeta.TitleID, + SoftwareStatusFilter: &expectStatus, + TeamFilter: teamID, + }) + require.NoError(t, err) + require.Len(t, hosts, 1) + require.Equal(t, host.ID, hosts[0].ID) + + // get software title includes status + summary, err := ds.GetSummaryHostSoftwareInstalls(ctx, installerMeta.InstallerID) + require.NoError(t, err) + require.Equal(t, fleet.SoftwareInstallerStatusSummary{ + Installed: 0, + Pending: 1, + Failed: 0, + }, *summary) }) } } diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index cf07aecacf..5494504dbb 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -2939,6 +2939,10 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { otherHost := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now()) opts := fleet.ListOptions{PerPage: 10, IncludeMetadata: true, OrderKey: "name", TestSecondaryOrderKey: "source"} + expectStatus := func(s fleet.SoftwareInstallerStatus) *fleet.SoftwareInstallerStatus { + return &s + } + // no software yet sw, meta, err := ds.ListHostSoftware(ctx, host.ID, false, opts) require.NoError(t, err) @@ -3157,7 +3161,7 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { expected["b"] = &fleet.HostSoftwareWithInstaller{ Name: "b", Source: "apps", - Status: &fleet.SoftwareInstallerPending, + Status: expectStatus(fleet.SoftwareInstallerPending), LastInstall: &fleet.HostSoftwareInstall{InstallUUID: "uuid1"}, PackageAvailableForInstall: ptr.String("installer-0.pkg"), InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ @@ -3167,14 +3171,14 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { expected["i0"] = &fleet.HostSoftwareWithInstaller{ Name: "i0", Source: "apps", - Status: &fleet.SoftwareInstallerInstalled, + Status: expectStatus(fleet.SoftwareInstallerInstalled), LastInstall: &fleet.HostSoftwareInstall{InstallUUID: "uuid2"}, PackageAvailableForInstall: ptr.String("installer-1.pkg"), } expected["i1"] = &fleet.HostSoftwareWithInstaller{ Name: "i1", Source: "apps", - Status: &fleet.SoftwareInstallerFailed, + Status: expectStatus(fleet.SoftwareInstallerFailed), LastInstall: &fleet.HostSoftwareInstall{InstallUUID: "uuid3"}, PackageAvailableForInstall: ptr.String("installer-2.pkg"), } @@ -3245,7 +3249,7 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { expected["b"] = &fleet.HostSoftwareWithInstaller{ Name: "b", Source: "apps", - Status: &fleet.SoftwareInstallerFailed, + Status: expectStatus(fleet.SoftwareInstallerFailed), LastInstall: &fleet.HostSoftwareInstall{InstallUUID: "uuid1"}, PackageAvailableForInstall: ptr.String("installer-0.pkg"), InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ @@ -3255,7 +3259,7 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { expected["i1"] = &fleet.HostSoftwareWithInstaller{ Name: "i1", Source: "apps", - Status: &fleet.SoftwareInstallerPending, + Status: expectStatus(fleet.SoftwareInstallerPending), LastInstall: &fleet.HostSoftwareInstall{InstallUUID: "uuid4"}, PackageAvailableForInstall: ptr.String("installer-2.pkg"), } diff --git a/server/datastore/mysql/software_titles.go b/server/datastore/mysql/software_titles.go index 0140dc0ffc..d8c47f850b 100644 --- a/server/datastore/mysql/software_titles.go +++ b/server/datastore/mysql/software_titles.go @@ -13,25 +13,30 @@ import ( ) func (ds *Datastore) SoftwareTitleByID(ctx context.Context, id uint, teamID *uint, tmFilter fleet.TeamFilter) (*fleet.SoftwareTitle, error) { - var teamFilter string + var teamFilter string // used to filter software titles host counts by team if teamID != nil { teamFilter = fmt.Sprintf("sthc.team_id = %d", *teamID) } else { teamFilter = ds.whereFilterGlobalOrTeamIDByTeams(tmFilter, "sthc") } + var tmID uint // used to filter software installers by team + if teamID != nil { + tmID = *teamID + } + selectSoftwareTitleStmt := fmt.Sprintf(` SELECT st.id, st.name, st.source, st.browser, - SUM(sthc.hosts_count) as hosts_count, + COALESCE(SUM(sthc.hosts_count), 0) as hosts_count, MAX(sthc.updated_at) as counts_updated_at FROM software_titles st -JOIN software_titles_host_counts sthc ON sthc.software_title_id = st.id -WHERE st.id = ? AND %s -AND sthc.hosts_count > 0 +LEFT JOIN software_titles_host_counts sthc ON sthc.software_title_id = st.id AND %s +WHERE st.id = ? +AND (sthc.hosts_count > 0 OR EXISTS (SELECT 1 FROM software_installers si WHERE si.title_id = st.id AND si.global_or_team_id = ?)) GROUP BY st.id, st.name, @@ -40,7 +45,7 @@ GROUP BY `, teamFilter, ) var title fleet.SoftwareTitle - if err := sqlx.GetContext(ctx, ds.reader(ctx), &title, selectSoftwareTitleStmt, id); err != nil { + if err := sqlx.GetContext(ctx, ds.reader(ctx), &title, selectSoftwareTitleStmt, id, tmID); err != nil { if err == sql.ErrNoRows { return nil, notFound("SoftwareTitle").WithID(id) } @@ -184,6 +189,8 @@ func spliceSecondaryOrderBySoftwareTitlesSQL(stmt string, opts fleet.ListOptions return strings.Replace(stmt, targetSubstr, targetSubstr+secondaryOrderBy, 1) } +// TODO: Does this need to be updated to include software installers? Otherwise, this list won't +// include software packages that haven't been installed on any hosts (see SoftwareTitleByID above). func selectSoftwareTitlesSQL(opt fleet.SoftwareTitleListOptions) (string, []any) { stmt := ` SELECT diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 23408ca269..ec626b157a 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1484,12 +1484,20 @@ type Datastore interface { // MatchOrCreateSoftwareInstaller matches or creates a new software installer. MatchOrCreateSoftwareInstaller(ctx context.Context, payload *UploadSoftwareInstallerPayload) (uint, error) - // GetSoftwareInstallerMetadata returns the software installer corresponding to the id. + // GetSoftwareInstallerMetadata returns the software installer corresponding to the installer id. GetSoftwareInstallerMetadata(ctx context.Context, id uint) (*SoftwareInstaller, error) + // GetSoftwareInstallerMetadataByTitleID returns the software installer corresponding to the specified + // team and title ids. + GetSoftwareInstallerMetadataByTeamAndTitleID(ctx context.Context, teamID *uint, titleID uint) (*SoftwareInstaller, error) + // DeleteSoftwareInstaller deletes the software installer corresponding to the id. DeleteSoftwareInstaller(ctx context.Context, id uint) error + // GetSoftwareInstallerContents returns the software install summary for the given + // software installer id. + GetSummaryHostSoftwareInstalls(ctx context.Context, installerID uint) (*SoftwareInstallerStatusSummary, error) + GetSoftwareInstallResults(ctx context.Context, resultsUUID string) (*HostSoftwareInstallerResult, error) // CleanupUnusedSoftwareInstallers will remove software installers that have diff --git a/server/fleet/hosts.go b/server/fleet/hosts.go index 0ed16f60c3..1c3bad6e6e 100644 --- a/server/fleet/hosts.go +++ b/server/fleet/hosts.go @@ -146,6 +146,9 @@ type HostListOptions struct { // use. This identifies a "software title" independent of the specific // version. SoftwareTitleIDFilter *uint + // SoftwareStatusFilter filters the hosts by the status of the software installer, if any, + // managed by Fleet. If specified, the SoftwareTitleIDFilter must also be specified. + SoftwareStatusFilter *SoftwareInstallerStatus OSIDFilter *uint OSNameFilter *string @@ -210,6 +213,7 @@ func (h HostListOptions) Empty() bool { h.SoftwareIDFilter == nil && h.SoftwareVersionIDFilter == nil && h.SoftwareTitleIDFilter == nil && + h.SoftwareStatusFilter == nil && h.OSIDFilter == nil && h.OSNameFilter == nil && h.OSVersionFilter == nil && diff --git a/server/fleet/software.go b/server/fleet/software.go index 9b15624920..d5afe4deab 100644 --- a/server/fleet/software.go +++ b/server/fleet/software.go @@ -157,7 +157,9 @@ type SoftwareTitle struct { Versions []SoftwareVersion `json:"versions" db:"-"` // CountsUpdatedAt is the timestamp when the hosts count // was last updated for that software title - CountsUpdatedAt time.Time `json:"-" db:"counts_updated_at"` + CountsUpdatedAt *time.Time `json:"-" db:"counts_updated_at"` + // SoftwarePackage is the software installer information for this title. + SoftwarePackage *SoftwareInstaller `json:"software_package" db:"-"` } type SoftwareTitleListOptions struct { diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index f6832aa419..5bf411dafa 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -89,6 +89,8 @@ type SoftwareInstaller struct { PostInstallScriptContentID *uint `json:"-" db:"post_install_script_content_id"` // StorageID is the unique identifier for the software package in the software installer store. StorageID string `json:"-" db:"storage_id"` + // Status is the status of the software installer package. + Status *SoftwareInstallerStatusSummary `json:"status,omitempty" db:"-"` // SoftwareTitle is the title of the software pointed installed by this installer. SoftwareTitle string `json:"-" db:"software_title"` } @@ -111,20 +113,36 @@ type SoftwareInstallerStatusSummary struct { // SoftwareInstallerStatus represents the status of a software installer package on a host. type SoftwareInstallerStatus string -var ( +const ( SoftwareInstallerPending SoftwareInstallerStatus = "pending" SoftwareInstallerFailed SoftwareInstallerStatus = "failed" SoftwareInstallerInstalled SoftwareInstallerStatus = "installed" ) +func (s SoftwareInstallerStatus) IsValid() bool { + switch s { + case + SoftwareInstallerFailed, + SoftwareInstallerInstalled, + SoftwareInstallerPending: + return true + default: + return false + } +} + // HostSoftwareInstaller represents a software installer package that has been installed on a host. type HostSoftwareInstallerResult struct { + // ID is the unique numerical ID of the result assigned by the datastore. + ID uint `json:"-" db:"id"` // InstallUUID is the unique identifier for the software install operation associated with the host. InstallUUID string `json:"install_uuid" db:"execution_id"` // SoftwareTitle is the title of the software. SoftwareTitle string `json:"software_title" db:"software_title"` // SoftwareVersion is the version of the software. SoftwareTitleID uint `json:"software_title_id" db:"software_title_id"` + // SoftwareInstallerID is the unique numerical ID of the software installer assigned by the datastore. + SoftwareInstallerID uint `json:"-" db:"software_installer_id"` // SoftwarePackage is the name of the software installer package. SoftwarePackage string `json:"software_package" db:"software_package"` // HostID is the ID of the host. @@ -142,6 +160,10 @@ type HostSoftwareInstallerResult struct { PreInstallQueryOutput string `json:"pre_install_query_output" db:"pre_install_query_output"` // PostInstallScriptOutput is the output of the post-install script on the host. PostInstallScriptOutput string `json:"post_install_script_output" db:"post_install_script_output"` + // CreatedAt is the time the software installer request was triggered. + CreatedAt time.Time `json:"created_at" db:"created_at"` + // UpdatedAt is the time the software installer request was last updated. + UpdatedAt *time.Time `json:"updated_at" db:"updated_at"` // HostTeamID is the team ID of the host on which this software install was attempted. This // field is not sent in the response, it is only used for internal authorization. HostTeamID *uint `json:"-" db:"host_team_id"` diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 3dc89b5976..a18a83fc85 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -941,8 +941,12 @@ type MatchOrCreateSoftwareInstallerFunc func(ctx context.Context, payload *fleet type GetSoftwareInstallerMetadataFunc func(ctx context.Context, id uint) (*fleet.SoftwareInstaller, error) +type GetSoftwareInstallerMetadataByTeamAndTitleIDFunc func(ctx context.Context, teamID *uint, titleID uint) (*fleet.SoftwareInstaller, error) + type DeleteSoftwareInstallerFunc func(ctx context.Context, id uint) error +type GetSummaryHostSoftwareInstallsFunc func(ctx context.Context, installerID uint) (*fleet.SoftwareInstallerStatusSummary, error) + type GetSoftwareInstallResultsFunc func(ctx context.Context, resultsUUID string) (*fleet.HostSoftwareInstallerResult, error) type CleanupUnusedSoftwareInstallersFunc func(ctx context.Context, softwareInstallStore fleet.SoftwareInstallerStore) error @@ -2331,9 +2335,15 @@ type DataStore struct { GetSoftwareInstallerMetadataFunc GetSoftwareInstallerMetadataFunc GetSoftwareInstallerMetadataFuncInvoked bool + GetSoftwareInstallerMetadataByTeamAndTitleIDFunc GetSoftwareInstallerMetadataByTeamAndTitleIDFunc + GetSoftwareInstallerMetadataByTeamAndTitleIDFuncInvoked bool + DeleteSoftwareInstallerFunc DeleteSoftwareInstallerFunc DeleteSoftwareInstallerFuncInvoked bool + GetSummaryHostSoftwareInstallsFunc GetSummaryHostSoftwareInstallsFunc + GetSummaryHostSoftwareInstallsFuncInvoked bool + GetSoftwareInstallResultsFunc GetSoftwareInstallResultsFunc GetSoftwareInstallResultsFuncInvoked bool @@ -5570,6 +5580,13 @@ func (s *DataStore) GetSoftwareInstallerMetadata(ctx context.Context, id uint) ( return s.GetSoftwareInstallerMetadataFunc(ctx, id) } +func (s *DataStore) GetSoftwareInstallerMetadataByTeamAndTitleID(ctx context.Context, teamID *uint, titleID uint) (*fleet.SoftwareInstaller, error) { + s.mu.Lock() + s.GetSoftwareInstallerMetadataByTeamAndTitleIDFuncInvoked = true + s.mu.Unlock() + return s.GetSoftwareInstallerMetadataByTeamAndTitleIDFunc(ctx, teamID, titleID) +} + func (s *DataStore) DeleteSoftwareInstaller(ctx context.Context, id uint) error { s.mu.Lock() s.DeleteSoftwareInstallerFuncInvoked = true @@ -5577,6 +5594,13 @@ func (s *DataStore) DeleteSoftwareInstaller(ctx context.Context, id uint) error return s.DeleteSoftwareInstallerFunc(ctx, id) } +func (s *DataStore) GetSummaryHostSoftwareInstalls(ctx context.Context, installerID uint) (*fleet.SoftwareInstallerStatusSummary, error) { + s.mu.Lock() + s.GetSummaryHostSoftwareInstallsFuncInvoked = true + s.mu.Unlock() + return s.GetSummaryHostSoftwareInstallsFunc(ctx, installerID) +} + func (s *DataStore) GetSoftwareInstallResults(ctx context.Context, resultsUUID string) (*fleet.HostSoftwareInstallerResult, error) { s.mu.Lock() s.GetSoftwareInstallResultsFuncInvoked = true diff --git a/server/service/hosts_test.go b/server/service/hosts_test.go index 8c556d1692..0f4bfcfcd2 100644 --- a/server/service/hosts_test.go +++ b/server/service/hosts_test.go @@ -1667,6 +1667,9 @@ func TestBulkOperationFilterValidation(t *testing.T) { return []*fleet.Host{}, nil } + // TODO(sarah): Future improvement to auto-generate a list of all possible filter values + // from `fleet.HostListOptions` and iterate to test that only a limited subset of filter (i.e. + // label_id, team_id, status, query) are allowed for bulk operations. tc := []struct { name string filters *map[string]interface{} diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index bc0f253021..2d4c330178 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -8740,7 +8740,7 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerNewInstallRequestPlatform } } -func (s *integrationMDMTestSuite) TestSoftwareInstallerNewInstallRequest() { +func (s *integrationMDMTestSuite) TestSoftwareInstallerHostRequests() { t := s.T() var resp installSoftwareResponse @@ -8785,15 +8785,12 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerNewInstallRequest() { resp = installSoftwareResponse{} s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/hosts/%d/software/install/%d", h.ID, titleID), nil, http.StatusAccepted, &resp) - // TODO(roberto): once we have endpoints to retrieve installers, - // request them using the orbit node key - // Get the results, should be pending - r := getHostSoftwareResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/hosts/%d/software", h.ID), nil, http.StatusOK, &r) - require.Len(t, r.Software, 1) - require.NotNil(t, r.Software[0].LastInstall) - installUUID := r.Software[0].LastInstall.InstallUUID + getHostSoftwareResp := getHostSoftwareResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/hosts/%d/software", h.ID), nil, http.StatusOK, &getHostSoftwareResp) + require.Len(t, getHostSoftwareResp.Software, 1) + require.NotNil(t, getHostSoftwareResp.Software[0].LastInstall) + installUUID := getHostSoftwareResp.Software[0].LastInstall.InstallUUID gsirr := getSoftwareInstallResultsResponse{} s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/software/install/results/%s", installUUID), nil, http.StatusOK, &gsirr) @@ -8802,6 +8799,107 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerNewInstallRequest() { results := gsirr.Results require.Equal(t, installUUID, results.InstallUUID) require.Equal(t, fleet.SoftwareInstallerPending, results.Status) + + // status is reflected in software title response + titleResp := getSoftwareTitleResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/software/titles/%d", titleID), nil, http.StatusOK, &titleResp) + // TODO: confirm expected behavior of the title response host counts (unspecified) + require.Zero(t, titleResp.SoftwareTitle.HostsCount) + require.Nil(t, titleResp.SoftwareTitle.CountsUpdatedAt) + require.NotNil(t, titleResp.SoftwareTitle.SoftwarePackage) + require.Equal(t, "ruby.deb", titleResp.SoftwareTitle.SoftwarePackage.Name) + require.NotNil(t, titleResp.SoftwareTitle.SoftwarePackage.Status) + require.Equal(t, fleet.SoftwareInstallerStatusSummary{ + Installed: 0, + Pending: 1, + Failed: 0, + }, *titleResp.SoftwareTitle.SoftwarePackage.Status) + + // status is reflected in list hosts responses and counts when filtering by software title and status + var listResp listHostsResponse + s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "pending", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID))) + require.Len(t, listResp.Hosts, 1) + require.Equal(t, h.ID, listResp.Hosts[0].ID) + + var countResp countHostsResponse + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "pending", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID))) + require.Equal(t, 1, countResp.Count) + + listResp = listHostsResponse{} + s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "failed", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID))) + require.Len(t, listResp.Hosts, 0) + + countResp = countHostsResponse{} + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "failed", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID))) + require.Equal(t, 0, countResp.Count) + + listResp = listHostsResponse{} + s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "installed", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID))) + require.Len(t, listResp.Hosts, 0) + + countResp = countHostsResponse{} + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "installed", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID))) + require.Equal(t, 0, countResp.Count) + + var labelResp createLabelResponse + s.DoJSON("POST", "/api/latest/fleet/labels", &createLabelRequest{fleet.LabelPayload{ + Name: "test", + Hosts: []string{h.Hostname}, + }}, http.StatusOK, &labelResp) + require.NotZero(t, labelResp.Label.ID) + + listResp = listHostsResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp) + require.Len(t, listResp.Hosts, 1) + require.Equal(t, h.ID, listResp.Hosts[0].ID) + + countResp = countHostsResponse{} + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "pending", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) + require.Equal(t, 1, countResp.Count) + + listResp = listHostsResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", "pending", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID))) + require.Len(t, listResp.Hosts, 1) + require.Equal(t, h.ID, listResp.Hosts[0].ID) + + countResp = countHostsResponse{} + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "pending", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) + require.Equal(t, 1, countResp.Count) + + listResp = listHostsResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", "installed", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID))) + require.Len(t, listResp.Hosts, 0) + + countResp = countHostsResponse{} + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "installed", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) + require.Equal(t, 0, countResp.Count) + + listResp = listHostsResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", "failed", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID))) + require.Len(t, listResp.Hosts, 0) + + countResp = countHostsResponse{} + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "failed", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) + require.Equal(t, 0, countResp.Count) + + // filter validations + r := s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "uninstalled") + require.Contains(t, extractServerErrorText(r.Body), "Invalid software_status") + r = s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "installed") + require.Contains(t, extractServerErrorText(r.Body), "Missing software_title_id") + r = s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "installed", "software_title_id", "1") + require.Contains(t, extractServerErrorText(r.Body), "Missing team_id") + r = s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "installed", "team_id", "1") + require.Contains(t, extractServerErrorText(r.Body), "Missing software_title_id") + r = s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "installed", "team_id", "1", "software_title_id", "1", "software_version_id", "1") + require.Contains(t, extractServerErrorText(r.Body), "Invalid parameters. The combination of software_version_id and software_title_id is not allowed.") + r = s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "installed", "team_id", "1", "software_title_id", "1", "software_id", "1") + require.Contains(t, extractServerErrorText(r.Body), "Invalid parameters. The combination of software_id and software_title_id is not allowed.") + + // TODO(roberto): once we have endpoints to retrieve installers, + // request them using the orbit node key + + // TODO(sarah): test other statuses once we have endpoints to set results via orbit } func (s *integrationMDMTestSuite) TestHostSoftwareInstallResult() { diff --git a/server/service/software_titles.go b/server/service/software_titles.go index 8cc264aee3..c9e3189802 100644 --- a/server/service/software_titles.go +++ b/server/service/software_titles.go @@ -39,8 +39,8 @@ func listSoftwareTitlesEndpoint(ctx context.Context, request interface{}, svc fl var latest time.Time for _, sw := range titles { - if !sw.CountsUpdatedAt.IsZero() && sw.CountsUpdatedAt.After(latest) { - latest = sw.CountsUpdatedAt + if sw.CountsUpdatedAt != nil && !sw.CountsUpdatedAt.IsZero() && sw.CountsUpdatedAt.After(latest) { + latest = *sw.CountsUpdatedAt } } listResp := listSoftwareTitlesResponse{ @@ -167,5 +167,25 @@ func (svc *Service) SoftwareTitleByID(ctx context.Context, id uint, teamID *uint return nil, ctxerr.Wrap(ctx, err, "getting software title by id") } + license, err := svc.License(ctx) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "get license") + } + if license.IsPremium() { + // add software installer data + meta, err := svc.ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, teamID, id) + if err != nil && !fleet.IsNotFound(err) { + return nil, ctxerr.Wrap(ctx, err, "get software installer metadata") + } + if meta != nil { + summary, err := svc.ds.GetSummaryHostSoftwareInstalls(ctx, meta.InstallerID) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "get software installer status summary") + } + meta.Status = summary + } + software.SoftwarePackage = meta + } + return software, nil } diff --git a/server/service/transport.go b/server/service/transport.go index 8db2a4b976..fadc537764 100644 --- a/server/service/transport.go +++ b/server/service/transport.go @@ -294,6 +294,30 @@ func hostListOptionsFromRequest(r *http.Request) (fleet.HostListOptions, error) hopt.SoftwareTitleIDFilter = &sid } + softwareStatus := fleet.SoftwareInstallerStatus(strings.ToLower(r.URL.Query().Get("software_status"))) + if softwareStatus != "" { + if !softwareStatus.IsValid() { + return hopt, ctxerr.Wrap( + r.Context(), badRequest(fmt.Sprintf("Invalid software_status: %s", softwareStatus)), + ) + } + if hopt.SoftwareTitleIDFilter == nil { + return hopt, ctxerr.Wrap( + r.Context(), badRequest( + "Missing software_title_id (it must be present when software_status is specified)", + ), + ) + } + if hopt.TeamFilter == nil { + return hopt, ctxerr.Wrap( + r.Context(), badRequest( + "Missing team_id (it must be present when software_status is specified)", + ), + ) + } + hopt.SoftwareStatusFilter = &softwareStatus + } + osID := r.URL.Query().Get("os_id") if osID != "" { id, err := strconv.ParseUint(osID, 10, 32) From b74b6078778ff1081ce81efa585dd08527c3d1fb Mon Sep 17 00:00:00 2001 From: Gabriel Hernandez Date: Thu, 9 May 2024 14:06:58 +0100 Subject: [PATCH 26/56] Update UI on software title details page for software packages (#18763) relates to #18327 implements UI on software title details page to supports the new software packages feature. This includes **new software package card on the software titles details page: ** ![image](https://github.com/fleetdm/fleet/assets/1153709/ade997db-dc43-428e-a50d-0e3b8b9a045b) **various modal for the package actions:** ![image](https://github.com/fleetdm/fleet/assets/1153709/a82df061-5bb7-40e0-9fb6-d96ea0da91e9) ![image](https://github.com/fleetdm/fleet/assets/1153709/9346bf46-5be4-4684-ab42-58f0a3089145) **ability to download the software package from download icon** - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [x] Manual QA for all new/changed functionality --- frontend/__mocks__/softwareMock.ts | 22 ++ frontend/components/Editor/Editor.tsx | 5 +- frontend/components/icons/Install.tsx | 15 +- frontend/components/icons/Settings.tsx | 32 +++ frontend/components/icons/index.ts | 2 + frontend/interfaces/software.ts | 16 +- .../AdvancedOptionsModal.tsx | 87 +++++++ .../AdvancedOptionsModal/_styles.scss | 13 ++ .../AdvancedOptionsModal/index.ts | 1 + .../DeleteSoftwareModal.tsx | 49 ++++ .../DeleteSoftwareModal/index.ts | 1 + .../SoftwarePackageCard.tsx | 216 ++++++++++++++++++ .../SoftwarePackageCard/_styles.scss | 69 ++++++ .../SoftwarePackageCard/index.ts | 1 + .../SoftwareTitleDetailsPage.tsx | 23 +- .../components/IconCell/IconCell.tsx | 4 +- frontend/services/entities/software.ts | 46 ++-- frontend/utilities/endpoints.ts | 4 +- 18 files changed, 584 insertions(+), 22 deletions(-) create mode 100644 frontend/components/icons/Settings.tsx create mode 100644 frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/AdvancedOptionsModal/AdvancedOptionsModal.tsx create mode 100644 frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/AdvancedOptionsModal/_styles.scss create mode 100644 frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/AdvancedOptionsModal/index.ts create mode 100644 frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/DeleteSoftwareModal/DeleteSoftwareModal.tsx create mode 100644 frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/DeleteSoftwareModal/index.ts create mode 100644 frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx create mode 100644 frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/_styles.scss create mode 100644 frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/index.ts diff --git a/frontend/__mocks__/softwareMock.ts b/frontend/__mocks__/softwareMock.ts index 663818cf26..aa808a4f8e 100644 --- a/frontend/__mocks__/softwareMock.ts +++ b/frontend/__mocks__/softwareMock.ts @@ -4,6 +4,7 @@ import { ISoftwareTitle, ISoftwareVulnerability, ISoftwareTitleVersion, + ISoftwarePackage, } from "interfaces/software"; import { ISoftwareTitlesResponse, @@ -148,3 +149,24 @@ export const createMockSoftwareVersionResponse = ( ): ISoftwareVersionResponse => { return { ...DEFAULT_SOFTWARE_VERSION_RESPONSE, ...overrides }; }; + +const DEFAULT_SOFTWAREPACKAGE_MOCK: ISoftwarePackage = { + name: "TestPackage-1.2.3.pkg", + version: "1.2.3", + uploaded_at: "2020-01-01T00:00:00.000Z", + install_script: "sudo installer -pkg /temp/FalconSensor-6.44.pkg -target /", + pre_install_query: "SELECT 1 FROM macos_profiles WHERE uuid='abc123';", + post_install_script: + "sudo /Applications/Falcon.app/Contents/Resources/falconctl license abc123", + status: { + installed: 1, + pending: 2, + failed: 3, + }, +}; + +export const createMockSoftwarePackage = ( + overrides?: Partial +) => { + return { ...DEFAULT_SOFTWAREPACKAGE_MOCK, ...overrides }; +}; diff --git a/frontend/components/Editor/Editor.tsx b/frontend/components/Editor/Editor.tsx index 4ab4fab32f..6b2fd919c6 100644 --- a/frontend/components/Editor/Editor.tsx +++ b/frontend/components/Editor/Editor.tsx @@ -8,6 +8,7 @@ interface IEditorProps { focus?: boolean; label?: string; error?: string | null; + readOnly?: boolean; /** * Help text to display below the editor. */ @@ -28,7 +29,7 @@ interface IEditorProps { name?: string; maxLines?: number; className?: string; - onChange: (value: string, event?: any) => void; + onChange?: (value: string, event?: any) => void; } /** @@ -45,6 +46,7 @@ const Editor = ({ focus, value, defaultValue, + readOnly = false, wrapEnabled = false, name = "editor", maxLines = 20, @@ -85,6 +87,7 @@ const Editor = ({ fontSize={14} theme="fleet" width="100%" + readOnly={readOnly} minLines={2} maxLines={maxLines} editorProps={{ $blockScrolling: Infinity }} diff --git a/frontend/components/icons/Install.tsx b/frontend/components/icons/Install.tsx index 768589d0c6..778b4fdc60 100644 --- a/frontend/components/icons/Install.tsx +++ b/frontend/components/icons/Install.tsx @@ -1,13 +1,24 @@ import React from "react"; + import { COLORS, Colors } from "styles/var/colors"; +import { ICON_SIZES, IconSizes } from "styles/var/icon_sizes"; interface IInstallProps { color?: Colors; + size?: IconSizes; } -const Install = ({ color = "ui-fleet-black-50" }: IInstallProps) => { +const Install = ({ + color = "ui-fleet-black-50", + size = "medium", +}: IInstallProps) => { return ( - + { + return ( + + + + ); +}; + +export default Settings; diff --git a/frontend/components/icons/index.ts b/frontend/components/icons/index.ts index 48f6903efc..0a19e3b1dc 100644 --- a/frontend/components/icons/index.ts +++ b/frontend/components/icons/index.ts @@ -57,6 +57,7 @@ import Download from "./Download"; import Upload from "./Upload"; import Refresh from "./Refresh"; import Install from "./Install"; +import Settings from "./Settings"; // a mapping of the usable names of icons to the icon source. export const ICON_MAP = { @@ -118,6 +119,7 @@ export const ICON_MAP = { upload: Upload, refresh: Refresh, install: Install, + settings: Settings, }; export type IconNames = keyof typeof ICON_MAP; diff --git a/frontend/interfaces/software.ts b/frontend/interfaces/software.ts index b96a3f5659..8023b25088 100644 --- a/frontend/interfaces/software.ts +++ b/frontend/interfaces/software.ts @@ -50,10 +50,24 @@ export interface ISoftwareTitleVersion { hosts_count?: number; } +export interface ISoftwarePackage { + name: string; + version: string; + uploaded_at: string; + install_script: string; + pre_install_query?: string; + post_install_script?: string; + status: { + installed: number; + pending: number; + failed: number; + }; +} + export interface ISoftwareTitle { id: number; name: string; - software_package: string | null; + software_package: ISoftwarePackage | string | null; versions_count: number; source: string; hosts_count: number; diff --git a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/AdvancedOptionsModal/AdvancedOptionsModal.tsx b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/AdvancedOptionsModal/AdvancedOptionsModal.tsx new file mode 100644 index 0000000000..30395a89d1 --- /dev/null +++ b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/AdvancedOptionsModal/AdvancedOptionsModal.tsx @@ -0,0 +1,87 @@ +import React from "react"; + +// @ts-ignore +import InputField from "components/forms/fields/InputField"; +import Modal from "components/Modal"; +import Button from "components/buttons/Button"; +import FleetAce from "components/FleetAce"; +import CustomLink from "components/CustomLink"; +import Editor from "components/Editor"; + +const baseClass = "advanced-options-modal"; + +interface IAdvancedOptionsModalProps { + installScript: string; + preInstallQuery?: string; + postInstallScript?: string; + onExit: () => void; +} + +const AdvancedOptionsModal = ({ + installScript, + preInstallQuery, + postInstallScript, + onExit, +}: IAdvancedOptionsModalProps) => { + return ( + + <> +

+ Advanced options are read-only. To change options, delete software and + add again. +

+
+ + {preInstallQuery && ( +
+ Pre-install condition: + + Software will be installed only if the{" "} + + + } + /> +
+ )} + {postInstallScript && ( +
+ Post-install script: + +
+ )} +
+
+ +
+ + + ); +}; + +export default AdvancedOptionsModal; diff --git a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/AdvancedOptionsModal/_styles.scss b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/AdvancedOptionsModal/_styles.scss new file mode 100644 index 0000000000..a63d438bdf --- /dev/null +++ b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/AdvancedOptionsModal/_styles.scss @@ -0,0 +1,13 @@ +.advanced-options-modal { + &__form-inputs { + display: flex; + flex-direction: column; + gap: $pad-large; + } + + &__input-field { + display: flex; + flex-direction: column; + gap: $pad-medium + } +} diff --git a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/AdvancedOptionsModal/index.ts b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/AdvancedOptionsModal/index.ts new file mode 100644 index 0000000000..79a369995f --- /dev/null +++ b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/AdvancedOptionsModal/index.ts @@ -0,0 +1 @@ +export { default } from "./AdvancedOptionsModal"; diff --git a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/DeleteSoftwareModal/DeleteSoftwareModal.tsx b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/DeleteSoftwareModal/DeleteSoftwareModal.tsx new file mode 100644 index 0000000000..91ab852b8e --- /dev/null +++ b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/DeleteSoftwareModal/DeleteSoftwareModal.tsx @@ -0,0 +1,49 @@ +import React, { useContext } from "react"; + +import softwareAPI from "services/entities/software"; +import { NotificationContext } from "context/notification"; + +import Modal from "components/Modal"; +import Button from "components/buttons/Button"; + +const baseClass = "delete-software-modal"; + +interface IDeleteSoftwareModalProps { + softwareId: number; + onExit: () => void; +} + +const DeleteSoftwareModal = ({ + softwareId, + onExit, +}: IDeleteSoftwareModalProps) => { + const { renderFlash } = useContext(NotificationContext); + + const onDeleteSoftware = async () => { + try { + await softwareAPI.deleteSoftwarePackage(softwareId); + renderFlash("success", "Software deleted successfully!"); + } catch { + renderFlash("error", "Couldn't delete. Please try again."); + } + onExit(); + }; + + return ( + + <> +

Software won't be uninstalled from existing hosts.

+
+ + +
+ +
+ ); +}; + +export default DeleteSoftwareModal; diff --git a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/DeleteSoftwareModal/index.ts b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/DeleteSoftwareModal/index.ts new file mode 100644 index 0000000000..e50bbbf812 --- /dev/null +++ b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/DeleteSoftwareModal/index.ts @@ -0,0 +1 @@ +export { default } from "./DeleteSoftwareModal"; diff --git a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx new file mode 100644 index 0000000000..e700b071c4 --- /dev/null +++ b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx @@ -0,0 +1,216 @@ +import React, { useContext, useState } from "react"; + +import endpoints from "utilities/endpoints"; +import software, { ISoftwarePackage } from "interfaces/software"; +import PATHS from "router/paths"; +import { AppContext } from "context/app"; +import { buildQueryStringFromParams } from "utilities/url"; +import { internationalTimeFormat } from "utilities/helpers"; +import { uploadedFromNow } from "utilities/date_format"; + +import Card from "components/Card"; +import Graphic from "components/Graphic"; +import TooltipWrapper from "components/TooltipWrapper"; +import DataSet from "components/DataSet"; +import Icon from "components/Icon"; +import Button from "components/buttons/Button"; + +import DeleteSoftwareModal from "../DeleteSoftwareModal"; +import AdvancedOptionsModal from "../AdvancedOptionsModal"; + +const baseClass = "software-package-card"; + +type IPackageInstallStatus = "installed" | "pending" | "failed"; +interface IStatusDisplayOption { + displayName: string; + iconName: "success" | "pending-outline" | "error"; + tooltip: string; +} + +const STATUS_DISPLAY_OPTIONS: Record< + IPackageInstallStatus, + IStatusDisplayOption +> = { + installed: { + displayName: "Installed", + iconName: "success", + tooltip: "Fleet installed software on these hosts.", + }, + pending: { + displayName: "Pending", + iconName: "pending-outline", + tooltip: "Fleet will install software when these hosts come online.", + }, + failed: { + displayName: "Failed", + iconName: "error", + tooltip: "Fleet failed to install software on these hosts.", + }, +}; + +interface IPackageStatusCountProps { + softwareId: number; + status: IPackageInstallStatus; + count: number; + teamId?: number; +} + +const PackageStatusCount = ({ + softwareId, + status, + count, + teamId, +}: IPackageStatusCountProps) => { + const displayData = STATUS_DISPLAY_OPTIONS[status]; + const linkUrl = `${PATHS.MANAGE_HOSTS}?${buildQueryStringFromParams({ + software_title_id: softwareId, + software_title_status: status, + team_id: teamId, + })}`; + return ( + +
+ + {displayData.displayName} +
+ + } + value={ + + {count} hosts + + } + /> + ); +}; + +interface ISoftwarePackageCardProps { + softwarePackage: ISoftwarePackage; + softwareId: number; + teamId?: number; +} + +const SoftwarePackageCard = ({ + softwarePackage, + softwareId, + teamId, +}: ISoftwarePackageCardProps) => { + const { + isGlobalAdmin, + isGlobalMaintainer, + isTeamAdmin, + isTeamMaintainer, + } = useContext(AppContext); + + const [showAdvancedOptionsModal, setShowAdvancedOptionsModal] = useState( + false + ); + const [showDeleteModal, setShowDeleteModal] = useState(false); + + const onAdvancedOptionsClick = () => { + setShowAdvancedOptionsModal(true); + }; + + const onDownloadClick = () => { + console.log("Download clicked"); + }; + + const onDeleteClick = () => { + setShowDeleteModal(true); + }; + + const showActions = + isGlobalAdmin || isGlobalMaintainer || isTeamAdmin || isTeamMaintainer; + + const downloadUrl = `/api${endpoints.SOFTWARE_PACKAGE( + softwareId + )}?${buildQueryStringFromParams({ alt: "media" })}`; + + return ( + +
+ {/* TODO: main-info could be a seperate component as its reused on a couple + pages already. Come back and pull this into a component */} +
+ +
+ + {softwarePackage.name} + + + Version {softwarePackage.version} • + + {uploadedFromNow(softwarePackage.uploaded_at)} + + +
+
+
+ + + +
+
+ {showActions && ( +
+ + {/* TODO: make a component for download icons */} + + + + +
+ )} + {showAdvancedOptionsModal && ( + setShowAdvancedOptionsModal(false)} + /> + )} + {showDeleteModal && ( + setShowDeleteModal(false)} + /> + )} +
+ ); +}; + +export default SoftwarePackageCard; diff --git a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/_styles.scss b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/_styles.scss new file mode 100644 index 0000000000..5c52ffe423 --- /dev/null +++ b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/_styles.scss @@ -0,0 +1,69 @@ +.software-package-card { + display: flex; + justify-content: space-between; + align-items: center; + + &__main-content { + display: flex; + align-items: center; + gap: $pad-xxlarge; + } + + &__main-info { + display: flex; + gap: $pad-medium; + } + + &__info { + display: flex; + flex-direction: column; + gap: $pad-xsmall; + } + + &__title { + font-size: $x-small; + font-weight: $bold; + } + + &__details { + font-size: $xx-small; + } + + &__package-statuses { + display: flex; + gap: $pad-xxlarge; + } + + &__status-title { + display: flex; + align-items: center; + gap: $pad-xsmall; + } + + &__status-count { + font-weight: normal; + } + + &__actions { + display: flex; + justify-content: flex-end; + gap: $pad-medium; + } + + &__download-icon { + display: flex; + justify-content: center; + width: 44px; + } + + @media (max-width: $break-md) { + align-items: flex-start; + + &__main-content { + display: flex; + flex-direction: column; + align-items: center; + gap: $pad-large; + } + } +} diff --git a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/index.ts b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/index.ts new file mode 100644 index 0000000000..bf345cb301 --- /dev/null +++ b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/index.ts @@ -0,0 +1 @@ +export { default } from "./SoftwarePackageCard"; diff --git a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwareTitleDetailsPage.tsx b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwareTitleDetailsPage.tsx index 007352f382..bc6f91a539 100644 --- a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwareTitleDetailsPage.tsx +++ b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwareTitleDetailsPage.tsx @@ -7,6 +7,7 @@ import { RouteComponentProps } from "react-router"; import { AxiosError } from "axios"; import useTeamIdParam from "hooks/useTeamIdParam"; +import { createMockSoftwarePackage } from "__mocks__/softwareMock"; import { AppContext } from "context/app"; @@ -16,7 +17,7 @@ import softwareAPI, { ISoftwareTitleResponse, IGetSoftwareTitleQueryKey, } from "services/entities/software"; - +import { APP_CONTEXT_ALL_TEAMS_ID } from "interfaces/team"; import { DEFAULT_USE_QUERY_OPTIONS } from "utilities/constants"; import Spinner from "components/Spinner"; @@ -27,6 +28,7 @@ import Card from "components/Card"; import SoftwareDetailsSummary from "../components/SoftwareDetailsSummary"; import SoftwareTitleDetailsTable from "./SoftwareTitleDetailsTable"; import DetailsNoHosts from "../components/DetailsNoHosts"; +import SoftwarePackageCard from "./SoftwarePackageCard"; const baseClass = "software-title-details-page"; @@ -45,7 +47,13 @@ const SoftwareTitleDetailsPage = ({ routeParams, location, }: ISoftwareTitleDetailsPageProps) => { - const { isPremiumTier, isOnGlobalTeam } = useContext(AppContext); + const { + isPremiumTier, + isOnGlobalTeam, + isTeamAdmin, + isTeamMaintainer, + isTeamObserver, + } = useContext(AppContext); const handlePageError = useErrorHandler(); // TODO: handle non integer values @@ -94,6 +102,10 @@ const SoftwareTitleDetailsPage = ({ [handleTeamChange] ); + const showPackageCard = + currentTeamId !== APP_CONTEXT_ALL_TEAMS_ID && + (isOnGlobalTeam || isTeamAdmin || isTeamMaintainer || isTeamObserver); + const renderContent = () => { if (isSoftwareTitleLoading) { return ; @@ -133,6 +145,13 @@ const SoftwareTitleDetailsPage = ({ name={softwareTitle.name} source={softwareTitle.source} /> + {showPackageCard && ( + + )} { +const IconCell = ({ iconName }: IIconCellProps) => { const tooltipID = uniqueId(); return ( diff --git a/frontend/services/entities/software.ts b/frontend/services/entities/software.ts index 725370636b..d9ebf6bc4c 100644 --- a/frontend/services/entities/software.ts +++ b/frontend/services/entities/software.ts @@ -11,6 +11,11 @@ import { } from "interfaces/software"; import { buildQueryStringFromParams, QueryParams } from "utilities/url"; import { IAddSoftwareFormData } from "pages/SoftwarePage/components/AddSoftwareForm/AddSoftwareForm"; +import { + createMockSoftwarePackage, + createMockSoftwareTitle, + createMockSoftwareTitleResponse, +} from "__mocks__/softwareMock"; export interface ISoftwareApiParams { page?: number; @@ -149,15 +154,6 @@ export default { return sendRequest("GET", path.concat(`?${queryString}`)); }, - getSoftwareById: async ( - softwareId: string - ): Promise => { - const { SOFTWARE } = endpoints; - const path = `${SOFTWARE}/${softwareId}`; - - return sendRequest("GET", path); - }, - getSoftwareTitles: ( params: ISoftwareApiParams ): Promise => { @@ -168,11 +164,22 @@ export default { return sendRequest("GET", path); }, - getSoftwareTitle: ({ softwareId, teamId }: IGetSoftwareTitleQueryParams) => { + getSoftwareTitle: ({ + softwareId, + teamId, + }: IGetSoftwareTitleQueryParams): Promise => { const endpoint = endpoints.SOFTWARE_TITLE(softwareId); const path = teamId ? `${endpoint}?team_id=${teamId}` : endpoint; + // return sendRequest("GET", path); - return sendRequest("GET", path); + // TODO: remove when we have API ready + return new Promise((resolve) => { + resolve({ + software_title: createMockSoftwareTitle({ + software_package: createMockSoftwarePackage(), + }), + }); + }); }, getSoftwareVersions: (params: ISoftwareApiParams) => { @@ -194,7 +201,7 @@ export default { }, addSoftwarePackage: (data: IAddSoftwareFormData, teamId?: number) => { - const { SOFTWARE_PACKAGE } = endpoints; + const { SOFTWARE_PACKAGE_ADD } = endpoints; if (!data.software) { throw new Error("Software package is required"); @@ -209,6 +216,19 @@ export default { formData.append("post_install_script", data.postInstallScript); teamId && formData.append("team_id", teamId.toString()); - return sendRequest("POST", SOFTWARE_PACKAGE, formData); + return sendRequest("POST", SOFTWARE_PACKAGE_ADD, formData); + }, + + deleteSoftwarePackage: (softwareId: number) => { + const { SOFTWARE_PACKAGE } = endpoints; + return sendRequest("DELETE", SOFTWARE_PACKAGE(softwareId)); + }, + + downloadSoftwarePackage: (softwareId: number) => { + const { SOFTWARE_PACKAGE } = endpoints; + const path = `${SOFTWARE_PACKAGE(softwareId)}?${buildQueryStringFromParams({ + alt: "media", + })}`; + return sendRequest("GET", path); }, }; diff --git a/frontend/utilities/endpoints.ts b/frontend/utilities/endpoints.ts index 8e75731f08..2245a69d15 100644 --- a/frontend/utilities/endpoints.ts +++ b/frontend/utilities/endpoints.ts @@ -128,7 +128,9 @@ export default { SOFTWARE_VERSIONS: `/${API_VERSION}/fleet/software/versions`, SOFTWARE_VERSION: (id: number) => `/${API_VERSION}/fleet/software/versions/${id}`, - SOFTWARE_PACKAGE: `/${API_VERSION}/fleet/software/package`, + SOFTWARE_PACKAGE_ADD: `/${API_VERSION}/fleet/software/package`, + SOFTWARE_PACKAGE: (id: number) => + `/${API_VERSION}/fleet/software/packages/${id}`, // AI endpoints AUTOFILL_POLICY: `/${API_VERSION}/fleet/autofill/policy`, From 643fc8314b3618aace724b9b981e4154ac699bb2 Mon Sep 17 00:00:00 2001 From: Dante Catalfamo <43040593+dantecatalfamo@users.noreply.github.com> Date: Thu, 9 May 2024 15:22:56 -0400 Subject: [PATCH 27/56] Orbit config receiver (#18518) New interface for adding periodic jobs that rely on notifications/config changes in Orbit. Previously if we wanted to have recurring checks in Orbit, we would add them into a chain of `GetConfig` calls. This call chain would be run periodically by one of the runners registered with the cli application framework. The new method to register `OrbitConfigReceivers` with the `OrbitClient`, and then register the orbit client itself with the application framework. Instead of having giving each fetcher an internal reference to the previous fetcher that it must call, the receiver is registered with the client and the new config is passed to the receiver. This is the old `GetConfig()` interface: ```go type OrbitConfigFetcher interface { GetConfig() (*fleet.OrbitConfig, error) } ``` This is the new `OrbitConfigReceiver` interface: ```go type OrbitConfigReceiver interface { Run(*OrbitConfig) error } ``` To register a new receiver, you call the `RegisterConfigReceiver` method on the client. ```go orbitClient.RegisterConfigReceiver(extRunner) ``` Downsides of the old method: - Spaghetti call chain setup - Cascading failure, of one fails, all after it fail - Run in series, one long function call holds up the rest - Anything that wants to restart orbit is added as a Runner to the application, meaning there could be several timers calling `GetConfig` and running the chain Benefits of the new method: - Clean `RegisterConfigReceiver` api, no call chaining required - Config receivers can be added at runtime - Isolated receivers, one failing call don't effect others - All calls are run in parallel in goroutines, no calls can hold up the rest - No more need for multiple runners, using a context cancel, any receiver can queue a call to restart orbit - Single point to handle errors and logging for all receivers - Panic recovery to stop orbit from crashing - Easier to test, configs are passed in and do not require a call chain This branch contains a little bit of code from the installer method I was working on because I branched it off of that. (oops) Not all code comments surrounding old `GetConfig()` methods have been fully updated yet Possible changes: - Update the interface to take a context, so we can let receivers know to exit early. I can imagine two cases for this: - The application is about to restart - We can set a timeout for how long receivers are allowed to take Closes #12662 --------- Co-authored-by: Martin Angers Co-authored-by: Roberto Dip --- orbit/cmd/orbit/orbit.go | 156 ++++-------- orbit/pkg/update/disk_encryption.go | 15 +- orbit/pkg/update/flag_runner.go | 155 +++--------- orbit/pkg/update/flag_runner_test.go | 35 ++- orbit/pkg/update/notifications.go | 119 ++++----- orbit/pkg/update/notifications_test.go | 319 ++++++++++--------------- orbit/pkg/update/nudge.go | 44 ++-- orbit/pkg/update/nudge_test.go | 47 ++-- orbit/pkg/update/swift_dialog.go | 22 +- orbit/pkg/update/swift_dialog_test.go | 6 +- server/fleet/orbit.go | 10 + server/service/orbit_client.go | 99 +++++++- server/service/orbit_client_test.go | 154 +++++++++++- 13 files changed, 580 insertions(+), 601 deletions(-) diff --git a/orbit/cmd/orbit/orbit.go b/orbit/cmd/orbit/orbit.go index 87bd73a56c..787d296e7f 100644 --- a/orbit/cmd/orbit/orbit.go +++ b/orbit/cmd/orbit/orbit.go @@ -803,81 +803,53 @@ func main() { windowsMDMEnrollmentCommandFrequency = time.Hour windowsMDMBitlockerCommandFrequency = time.Hour ) - configFetcher := update.ApplyRenewEnrollmentProfileConfigFetcherMiddleware(orbitClient, renewEnrollmentProfileCommandFrequency, fleetURL) - configFetcher, scriptsEnabledFn := update.ApplyRunScriptsConfigFetcherMiddleware( - configFetcher, c.Bool("enable-scripts"), orbitClient, + + orbitClient.RegisterConfigReceiver(update.ApplyRenewEnrollmentProfileConfigFetcherMiddleware(orbitClient, renewEnrollmentProfileCommandFrequency, fleetURL)) + scriptConfigReceiver, scriptsEnabledFn := update.ApplyRunScriptsConfigFetcherMiddleware( + c.Bool("enable-scripts"), orbitClient, ) + orbitClient.RegisterConfigReceiver(scriptConfigReceiver) switch runtime.GOOS { case "darwin": // add middleware to handle nudge installation and updates const nudgeLaunchInterval = 30 * time.Minute - configFetcher = update.ApplyNudgeConfigFetcherMiddleware(configFetcher, update.NudgeConfigFetcherOptions{ + orbitClient.RegisterConfigReceiver(update.ApplyNudgeConfigReceiverMiddleware(update.NudgeConfigFetcherOptions{ UpdateRunner: updateRunner, RootDir: c.String("root-dir"), Interval: nudgeLaunchInterval, - }) - - configFetcher = update.ApplyDiskEncryptionRunnerMiddleware(configFetcher) - configFetcher = update.ApplySwiftDialogDownloaderMiddleware(configFetcher, updateRunner) + })) + orbitClient.RegisterConfigReceiver(update.ApplyDiskEncryptionRunnerMiddleware()) + orbitClient.RegisterConfigReceiver(update.ApplySwiftDialogDownloaderMiddleware(updateRunner)) case "windows": - configFetcher = update.ApplyWindowsMDMEnrollmentFetcherMiddleware(configFetcher, windowsMDMEnrollmentCommandFrequency, orbitHostInfo.HardwareUUID, orbitClient) - configFetcher = update.ApplyWindowsMDMBitlockerFetcherMiddleware(configFetcher, windowsMDMBitlockerCommandFrequency, orbitClient) + orbitClient.RegisterConfigReceiver(update.ApplyWindowsMDMEnrollmentFetcherMiddleware(windowsMDMEnrollmentCommandFrequency, orbitHostInfo.HardwareUUID, orbitClient)) + orbitClient.RegisterConfigReceiver(update.ApplyWindowsMDMBitlockerFetcherMiddleware(windowsMDMBitlockerCommandFrequency, orbitClient)) } - const orbitFlagsUpdateInterval = 30 * time.Second - flagRunner := update.NewFlagRunner(configFetcher, update.FlagUpdateOptions{ - CheckInterval: orbitFlagsUpdateInterval, - RootDir: c.String("root-dir"), + flagUpdateReciver := update.NewFlagReceiver(orbitClient.ReceiverUpdateCancelFunc, update.FlagUpdateOptions{ + RootDir: c.String("root-dir"), }) - // Try performing a flags update to use latest configured osquery flags from get-go. - // This also takes care of populating the server's capabilities as it calls the orbit - // config endpoint. - if _, err := flagRunner.DoFlagsUpdate(); err != nil { - // Just log, OK to continue, since flagRunner will retry - // in flagRunner.Execute. - log.Debug().Err(err).Msg("initial flags update failed") - } - g.Add(flagRunner.Execute, flagRunner.Interrupt) + orbitClient.RegisterConfigReceiver(flagUpdateReciver) if !c.Bool("disable-updates") { - const serverOverridesInterval = 30 * time.Second - serverOverridesRunner := newServerOverridesRunner( - configFetcher, + serverOverridesReceiver := newServerOverridesReceiver( c.String("root-dir"), - serverOverridesInterval, fallbackServerOverridesConfig{ OsquerydPath: osquerydPath, DesktopPath: desktopPath, }, c.Bool("fleet-desktop"), + orbitClient.ReceiverUpdateCancelFunc, ) - // Perform initial run to update overrides as soon as possible. - didUpdate, err := serverOverridesRunner.run() - if err != nil { - // Just log, OK to continue, since serverOverridesRunner will retry - // in serverOverridesRunner.Execute. - log.Debug().Err(err).Msg("initial flags update failed") - } - if didUpdate { - log.Info().Msg("exiting due to early update of server overrides") - return nil - } - g.Add(serverOverridesRunner.Execute, serverOverridesRunner.Interrupt) + + orbitClient.RegisterConfigReceiver(serverOverridesReceiver) } // only setup extensions autoupdate if we have enabled updates // for extensions autoupdate, we can only proceed after orbit is enrolled in fleet // and all relevant things for it (like certs, enroll secrets, tls proxy, etc) is configured if !c.Bool("disable-updates") || c.Bool("dev-mode") { - const orbitExtensionUpdateInterval = 60 * time.Second - extRunner := update.NewExtensionConfigUpdateRunner(configFetcher, update.ExtensionUpdateOptions{ - CheckInterval: orbitExtensionUpdateInterval, - RootDir: c.String("root-dir"), - }, updateRunner) - - if _, err := extRunner.DoExtensionConfigUpdate(); err != nil { - // just log, OK to continue since this will get retry - logging.LogErrIfEnvNotSet(constant.SilenceEnrollLogErrorEnvVar, err, "initial update to fetch extensions from /config API failed") - } + extRunner := update.NewExtensionConfigUpdateRunner(update.ExtensionUpdateOptions{ + RootDir: c.String("root-dir"), + }, updateRunner, orbitClient.ReceiverUpdateCancelFunc) // call UpdateAction on the updateRunner after we have fetched extensions from Fleet _, err := updateRunner.UpdateAction() @@ -904,9 +876,16 @@ func main() { default: logging.LogErrIfEnvNotSet(constant.SilenceEnrollLogErrorEnvVar, err, "error with extensions.load file at "+extensionAutoLoadFile) } - g.Add(extRunner.Execute, extRunner.Interrupt) + + orbitClient.RegisterConfigReceiver(extRunner) } + if err := orbitClient.RunConfigReceivers(); err != nil { + log.Error().Msgf("failed initial config fetch: %s", err) + } + + g.Add(orbitClient.ExecuteConfigReceivers, orbitClient.InterruptConfigReceivers) + var trw *token.ReadWriter if c.Bool("fleet-desktop") { trw = token.NewReadWriter(filepath.Join(c.String("root-dir"), "identifier")) @@ -1663,87 +1642,50 @@ func writeSecret(enrollSecret string, orbitRoot string) error { // serverOverridesRunner is a oklog.Group runner that polls for configuration overrides from Fleet. type serverOverridesRunner struct { - configFetcher update.OrbitConfigFetcher - interval time.Duration - rootDir string - fallbackCfg fallbackServerOverridesConfig - desktopEnabled bool - cancel chan struct{} + rootDir string + fallbackCfg fallbackServerOverridesConfig + desktopEnabled bool + cancel chan struct{} + queueOrbitRestart context.CancelFunc } -// newServerOverridesRunner creates a runner for updating server overrides configuration with values fetched from Fleet. -func newServerOverridesRunner( - configFetcher update.OrbitConfigFetcher, +// newServerOverridesReveiver creates a runner for updating server overrides configuration with values fetched from Fleet. +func newServerOverridesReceiver( rootDir string, - interval time.Duration, fallbackCfg fallbackServerOverridesConfig, desktopEnabled bool, + queueOrbitRestart context.CancelFunc, ) *serverOverridesRunner { return &serverOverridesRunner{ - configFetcher: configFetcher, - interval: interval, - rootDir: rootDir, - fallbackCfg: fallbackCfg, - desktopEnabled: desktopEnabled, - cancel: make(chan struct{}), + rootDir: rootDir, + fallbackCfg: fallbackCfg, + desktopEnabled: desktopEnabled, + cancel: make(chan struct{}), + queueOrbitRestart: queueOrbitRestart, } } -// Execute starts the loop that polls for server overrides configuration from Fleet. -func (r *serverOverridesRunner) Execute() error { - log.Debug().Msg("starting server overrides runner") - - ticker := time.NewTicker(r.interval) - defer ticker.Stop() - - for { - select { - case <-r.cancel: - return nil - case <-ticker.C: - log.Debug().Msg("calling server overrides run") - didUpdate, err := r.run() - if err != nil { - logging.LogErrIfEnvNotSet(constant.SilenceEnrollLogErrorEnvVar, err, "server overrides run failed") - } - if didUpdate { - log.Info().Msg("server overrides updated, exiting") - return nil - } - } - } -} - -// Interrupt is the oklog/run interrupt method that stops orbit when interrupt is received -func (r *serverOverridesRunner) Interrupt(err error) { - close(r.cancel) - log.Error().Err(err).Msg("interrupt for server overrides runner") -} - -func (r *serverOverridesRunner) run() (bool, error) { +func (r *serverOverridesRunner) Run(orbitCfg *fleet.OrbitConfig) error { overrideCfg, err := loadServerOverrides(r.rootDir) if err != nil { - return false, err + return err } - orbitCfg, err := r.configFetcher.GetConfig() - if err != nil { - return false, err - } if orbitCfg.UpdateChannels == nil { // Server is not setting or doesn't know of // this feature (old server version), so nothing to do. - return false, nil + return nil } if cfgsDiffer(overrideCfg, orbitCfg, r.desktopEnabled) { if err := r.updateServerOverrides(orbitCfg); err != nil { - return false, err + return err } - return true, nil + r.queueOrbitRestart() + return nil } - return false, nil + return nil } // cfgsDiffer returns whether the local server overrides differ from the fetched remotely. diff --git a/orbit/pkg/update/disk_encryption.go b/orbit/pkg/update/disk_encryption.go index e922cfdfdc..ae09f386d6 100644 --- a/orbit/pkg/update/disk_encryption.go +++ b/orbit/pkg/update/disk_encryption.go @@ -11,21 +11,14 @@ import ( const maxRetries = 2 type DiskEncryptionRunner struct { - fetcher OrbitConfigFetcher isRunning atomic.Bool } -func ApplyDiskEncryptionRunnerMiddleware(f OrbitConfigFetcher) *DiskEncryptionRunner { - return &DiskEncryptionRunner{fetcher: f} +func ApplyDiskEncryptionRunnerMiddleware() fleet.OrbitConfigReceiver { + return &DiskEncryptionRunner{} } -func (d *DiskEncryptionRunner) GetConfig() (*fleet.OrbitConfig, error) { - cfg, err := d.fetcher.GetConfig() - if err != nil { - log.Debug().Err(err).Msg("calling GetConfig from DiskEncryptionFetcher") - return nil, err - } - +func (d *DiskEncryptionRunner) Run(cfg *fleet.OrbitConfig) error { log.Debug().Msgf("running disk encryption fetcher middleware, notification: %v, isIdle: %v", cfg.Notifications.RotateDiskEncryptionKey, d.isRunning.Load()) if cfg.Notifications.RotateDiskEncryptionKey && !d.isRunning.Swap(true) { @@ -37,5 +30,5 @@ func (d *DiskEncryptionRunner) GetConfig() (*fleet.OrbitConfig, error) { }() } - return cfg, nil + return nil } diff --git a/orbit/pkg/update/flag_runner.go b/orbit/pkg/update/flag_runner.go index 33fecc380e..19acdccfe8 100644 --- a/orbit/pkg/update/flag_runner.go +++ b/orbit/pkg/update/flag_runner.go @@ -1,6 +1,7 @@ package update import ( + "context" "encoding/json" "errors" "fmt" @@ -10,10 +11,8 @@ import ( "runtime" "strconv" "strings" - "time" "github.com/fleetdm/fleet/v4/orbit/pkg/constant" - "github.com/fleetdm/fleet/v4/orbit/pkg/logging" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/rs/zerolog/log" ) @@ -24,103 +23,64 @@ import ( // It uses an OrbitConfigFetcher (which may be the OrbitClient with additional middleware), along // with FlagUpdateOptions to connect to Fleet type FlagRunner struct { - configFetcher OrbitConfigFetcher - opt FlagUpdateOptions - cancel chan struct{} + queueOrbitRestart context.CancelFunc + opt FlagUpdateOptions } // FlagUpdateOptions is options provided for the flag update runner type FlagUpdateOptions struct { - // CheckInterval is the interval to check for updates - CheckInterval time.Duration // RootDir is the root directory for orbit state RootDir string } // NewFlagRunner creates a new runner with provided options // The runner must be started with Execute -func NewFlagRunner(configFetcher OrbitConfigFetcher, opt FlagUpdateOptions) *FlagRunner { +func NewFlagReceiver(queueOrbitRestart context.CancelFunc, opt FlagUpdateOptions) *FlagRunner { return &FlagRunner{ - configFetcher: configFetcher, - opt: opt, - cancel: make(chan struct{}), + queueOrbitRestart: queueOrbitRestart, + opt: opt, } } -// Execute starts the loop checking for updates -func (r *FlagRunner) Execute() error { - log.Debug().Msg("starting flag updater") - - ticker := time.NewTicker(r.opt.CheckInterval) - defer ticker.Stop() - - for { - select { - case <-r.cancel: - return nil - case <-ticker.C: - log.Debug().Msg("calling flags update") - didUpdate, err := r.DoFlagsUpdate() - if err != nil { - logging.LogErrIfEnvNotSet(constant.SilenceEnrollLogErrorEnvVar, err, "flags updates failed") - } - if didUpdate { - log.Info().Msg("flags updated, exiting") - return nil - } - ticker.Reset(r.opt.CheckInterval) - } - } -} - -// Interrupt is the oklog/run interrupt method that stops orbit when interrupt is received -func (r *FlagRunner) Interrupt(err error) { - close(r.cancel) - log.Error().Err(err).Msg("interrupt for flags updater") -} - // DoFlagsUpdate checks for update of flags from Fleet // It gets the flags from the Fleet server, and compares them to locally stored flagfile (if it exists) // If the flag comparison from disk and server are not equal, it writes the flags to disk, and returns true -func (r *FlagRunner) DoFlagsUpdate() (bool, error) { +func (r *FlagRunner) Run(config *fleet.OrbitConfig) error { flagFileExists := true // first off try and read osquery.flags from disk osqueryFlagMapFromFile, err := readFlagFile(r.opt.RootDir) if err != nil { if !errors.Is(err, os.ErrNotExist) { - return false, err + return err } // flag file may not exist on disk on first "boot" flagFileExists = false } - // next GetConfig from Fleet API - config, err := r.configFetcher.GetConfig() - if err != nil { - return false, fmt.Errorf("error getting flags from fleet: %w", err) - } if len(config.Flags) == 0 { // command_line_flags not set in YAML, nothing to do - return false, nil + return nil } osqueryFlagMapFromFleet, err := getFlagsFromJSON(config.Flags) if err != nil { - return false, fmt.Errorf("error parsing flags: %w", err) + return fmt.Errorf("error parsing flags: %w", err) } // compare both flags, if they are equal, nothing to do if flagFileExists && reflect.DeepEqual(osqueryFlagMapFromFile, osqueryFlagMapFromFleet) { - return false, nil + return nil } // flags are not equal, write the fleet flags to disk err = writeFlagFile(r.opt.RootDir, osqueryFlagMapFromFleet) if err != nil { - return false, fmt.Errorf("error writing flags to disk: %w", err) + return fmt.Errorf("error writing flags to disk: %w", err) } - return true, nil + + r.queueOrbitRestart() + return nil } // ExtensionRunner is a specialized runner to periodically check and update flags from Fleet @@ -129,76 +89,33 @@ func (r *FlagRunner) DoFlagsUpdate() (bool, error) { // It uses an an OrbitConfigFetcher (which may be the OrbitClient with additional middleware), along // with ExtensionUpdateOptions and updateRunner to connect to Fleet. type ExtensionRunner struct { - configFetcher OrbitConfigFetcher - opt ExtensionUpdateOptions - cancel chan struct{} - updateRunner *Runner + opt ExtensionUpdateOptions + updateRunner *Runner + queueOrbitRestart context.CancelFunc } // ExtensionUpdateOptions is options provided for the extensions fetch/update runner type ExtensionUpdateOptions struct { - // CheckInterval is the interval to check for updates - CheckInterval time.Duration // RootDir is the root directory for orbit state RootDir string } // NewExtensionConfigUpdateRunner creates a new runner with provided options // The runner must be started with Execute -func NewExtensionConfigUpdateRunner(configFetcher OrbitConfigFetcher, opt ExtensionUpdateOptions, updateRunner *Runner) *ExtensionRunner { +func NewExtensionConfigUpdateRunner(opt ExtensionUpdateOptions, updateRunner *Runner, queueOrbitRestart context.CancelFunc) *ExtensionRunner { return &ExtensionRunner{ - configFetcher: configFetcher, - opt: opt, - cancel: make(chan struct{}), - updateRunner: updateRunner, + opt: opt, + updateRunner: updateRunner, + queueOrbitRestart: queueOrbitRestart, } } -// Execute starts the loop checking for updates -func (r *ExtensionRunner) Execute() error { - log.Debug().Msg("starting extension runner") - - ticker := time.NewTicker(r.opt.CheckInterval) - defer ticker.Stop() - - for { - select { - case <-r.cancel: - return nil - case <-ticker.C: - log.Debug().Msg("calling /config API to fetch/update extensions") - extensionsCleared, err := r.DoExtensionConfigUpdate() - if err != nil { - logging.LogErrIfEnvNotSet(constant.SilenceEnrollLogErrorEnvVar, err, "ext update failed") - } - if extensionsCleared { - log.Info().Msg("extensions were cleared on the server") - return nil - } - } - ticker.Reset(r.opt.CheckInterval) - } -} - -// Interrupt is the oklog/run interrupt method that stops orbit when interrupt is received -func (r *ExtensionRunner) Interrupt(err error) { - close(r.cancel) - log.Error().Err(err).Msg(("interrupt extension runner")) -} - // DoExtensionConfigUpdate calls the /config API endpoint to grab extensions from Fleet // It parses the extensions, computes the local hash, and writes the binary path to extension.load file // // It returns a (bool, error), where bool indicates whether orbit should restart // It only returns (true, nil) when extensions were previously configured and now are cleared -func (r *ExtensionRunner) DoExtensionConfigUpdate() (bool, error) { - // call "/config" API endpoint to grab orbit configs from Fleet - config, err := r.configFetcher.GetConfig() - if err != nil { - // we do not want orbit to restart - return false, fmt.Errorf("extensionsUpdate: error getting extensions config from fleet: %w", err) - } - +func (r *ExtensionRunner) Run(config *fleet.OrbitConfig) error { extensionAutoLoadFile := filepath.Join(r.opt.RootDir, "extensions.load") if len(config.Extensions) == 0 { // Extensions from Fleet is empty @@ -210,7 +127,7 @@ func (r *ExtensionRunner) DoExtensionConfigUpdate() (bool, error) { case errors.Is(err, os.ErrNotExist): log.Debug().Msg(extensionAutoLoadFile + " not found, nothing to update") // we do not want orbit to restart - return false, nil + return nil case err == nil: // handle case 2: create/truncate the extensions.load file and let the runner interrupt, so that // osquery can't startup without the extensions that were previously loaded @@ -219,27 +136,29 @@ func (r *ExtensionRunner) DoExtensionConfigUpdate() (bool, error) { err := os.WriteFile(extensionAutoLoadFile, []byte(""), constant.DefaultFileMode) if err != nil { // we do not want orbit to restart - return false, fmt.Errorf("extensionsUpdate: error creating file %s, %w", extensionAutoLoadFile, err) + return fmt.Errorf("extensionsUpdate: error creating file %s, %w", extensionAutoLoadFile, err) } // we want to return true here, and restart with the empty extensions.load file - // so that we "unload" the previously loaded extensions - return true, nil + // so that we "unload" the previously loaded + // extensions + r.queueOrbitRestart() + return nil } // we do not want orbit to restart - return false, nil + return nil default: // we do not want orbit to restart, just log the error - return false, fmt.Errorf("stat file: %s", extensionAutoLoadFile) + return fmt.Errorf("stat file: %s", extensionAutoLoadFile) } } log.Debug().Str("extensions", string(config.Extensions)).Msg("received extensions configuration") var extensions fleet.Extensions - err = json.Unmarshal(config.Extensions, &extensions) + err := json.Unmarshal(config.Extensions, &extensions) if err != nil { // we do not want orbit to restart - return false, fmt.Errorf("error unmarshing json extensions config from fleet: %w", err) + return fmt.Errorf("error unmarshing json extensions config from fleet: %w", err) } // Filter out extensions not targeted to this OS. @@ -282,23 +201,23 @@ func (r *ExtensionRunner) DoExtensionConfigUpdate() (bool, error) { if err := r.updateRunner.updater.UpdateMetadata(); err != nil { // Consider this a non-fatal error since it will be common to be offline // or otherwise unable to retrieve the metadata. - return false, fmt.Errorf("update metadata: %w", err) + return fmt.Errorf("update metadata: %w", err) } if err := r.updateRunner.StoreLocalHash(targetName); err != nil { // we do not want orbit to restart - return false, fmt.Errorf("unable to lookup metadata for target: %s, %w", targetName, err) + return fmt.Errorf("unable to lookup metadata for target: %s, %w", targetName, err) } sb.WriteString(path + "\n") } if err := os.WriteFile(extensionAutoLoadFile, []byte(sb.String()), constant.DefaultFileMode); err != nil { - return false, fmt.Errorf("error writing extensions autoload file: %w", err) + return fmt.Errorf("error writing extensions autoload file: %w", err) } // we do not want orbit to restart // runner.UpdateAction() will fetch the new targets and restart for us if needed - return false, nil + return nil } // getFlagsFromJSON converts a json document of the form diff --git a/orbit/pkg/update/flag_runner_test.go b/orbit/pkg/update/flag_runner_test.go index 2e333700be..d5e366ab6b 100644 --- a/orbit/pkg/update/flag_runner_test.go +++ b/orbit/pkg/update/flag_runner_test.go @@ -77,14 +77,6 @@ func touchFile(t *testing.T, name string) { require.NoError(t, file.Close()) } -type dummyConfigFetcher struct { - cfg *fleet.OrbitConfig -} - -func (d *dummyConfigFetcher) GetConfig() (*fleet.OrbitConfig, error) { - return d.cfg, nil -} - // TestDoFlagsUpdateWithEmptyFlags tests the scenario of Fleet flag `command_line_flags` // being set to an empty JSON document `{}` and Orbit osquery.flags file being // an empty file. Such scenario should trigger no update of flags. @@ -93,32 +85,37 @@ func TestDoFlagsUpdateWithEmptyFlags(t *testing.T) { osqueryFlagsFile := filepath.Join(rootDir, "osquery.flags") touchFile(t, osqueryFlagsFile) - dcf := dummyConfigFetcher{cfg: &fleet.OrbitConfig{ + testConfig := &fleet.OrbitConfig{ Flags: json.RawMessage("{}"), - }} - fr := NewFlagRunner(&dcf, FlagUpdateOptions{ + } + + var restartQueued bool + queueOrbitRestart := func() { restartQueued = true } + + fr := NewFlagReceiver(queueOrbitRestart, FlagUpdateOptions{ RootDir: rootDir, }) - needsUpdate, err := fr.DoFlagsUpdate() + err := fr.Run(testConfig) require.NoError(t, err) - require.False(t, needsUpdate) + require.False(t, restartQueued) // Non-empty fleet flags and osquery.flags has empty flags. - dcf.cfg = &fleet.OrbitConfig{ + testConfig = &fleet.OrbitConfig{ Flags: json.RawMessage(`{"--verbose": true}`), } - needsUpdate, err = fr.DoFlagsUpdate() + err = fr.Run(testConfig) require.NoError(t, err) - require.True(t, needsUpdate) + require.True(t, restartQueued) // Empty Fleet flags and osquery.flags has non-empty flags. - dcf.cfg = &fleet.OrbitConfig{ + restartQueued = false + testConfig = &fleet.OrbitConfig{ Flags: json.RawMessage("{}"), } err = os.WriteFile(osqueryFlagsFile, []byte("--verbose=true\n"), 0o644) require.NoError(t, err) - needsUpdate, err = fr.DoFlagsUpdate() + err = fr.Run(testConfig) require.NoError(t, err) - require.True(t, needsUpdate) + require.True(t, restartQueued) } diff --git a/orbit/pkg/update/notifications.go b/orbit/pkg/update/notifications.go index febb8cc054..c185ee64bd 100644 --- a/orbit/pkg/update/notifications.go +++ b/orbit/pkg/update/notifications.go @@ -20,7 +20,7 @@ type checkEnrollmentFunc func() (bool, string, error) type checkAssignedEnrollmentProfileFunc func(url string) error -// renewEnrollmentProfileConfigFetcher is a kind of middleware that wraps an +// renewEnrollmentProfileConfigReceiver is a kind of middleware that wraps an // OrbitConfigFetcher and detects if the fleet server sent a notification to // renew the enrollment profile. If so, it runs the command (as root) to // bootstrap the renewal of the profile on the device (the user still needs to @@ -28,10 +28,7 @@ type checkAssignedEnrollmentProfileFunc func(url string) error // // It ensures only one renewal command is executed at any given time, and that // it doesn't re-execute the command until a certain amount of time has passed. -type renewEnrollmentProfileConfigFetcher struct { - // Fetcher is the OrbitConfigFetcher that will be wrapped. It is responsible - // for actually returning the orbit configuration or an error. - Fetcher OrbitConfigFetcher +type renewEnrollmentProfileConfigReceiver struct { // Frequency is the minimum amount of time that must pass between two // executions of the profile renewal command. Frequency time.Duration @@ -54,17 +51,12 @@ type renewEnrollmentProfileConfigFetcher struct { fleetURL string } -func ApplyRenewEnrollmentProfileConfigFetcherMiddleware(fetcher OrbitConfigFetcher, frequency time.Duration, fleetURL string) OrbitConfigFetcher { - return &renewEnrollmentProfileConfigFetcher{Fetcher: fetcher, Frequency: frequency, fleetURL: fleetURL} +func ApplyRenewEnrollmentProfileConfigFetcherMiddleware(fetcher OrbitConfigFetcher, frequency time.Duration, fleetURL string) fleet.OrbitConfigReceiver { + return &renewEnrollmentProfileConfigReceiver{Frequency: frequency, fleetURL: fleetURL} } -// GetConfig calls the wrapped Fetcher's GetConfig method, and if the fleet -// server set the renew enrollment profile flag to true, executes the command -// to renew the enrollment profile. -func (h *renewEnrollmentProfileConfigFetcher) GetConfig() (*fleet.OrbitConfig, error) { - cfg, err := h.Fetcher.GetConfig() - - if err == nil && cfg.Notifications.RenewEnrollmentProfile { +func (h *renewEnrollmentProfileConfigReceiver) Run(config *fleet.OrbitConfig) error { + if config.Notifications.RenewEnrollmentProfile { if h.cmdMu.TryLock() { defer h.cmdMu.Unlock() @@ -83,12 +75,12 @@ func (h *renewEnrollmentProfileConfigFetcher) GetConfig() (*fleet.OrbitConfig, e enrolled, mdmServerURL, err := enrollFn() if err != nil { log.Error().Err(err).Msg("fetching enrollment status") - return cfg, nil + return nil } if enrolled { log.Info().Msgf("a request to renew the enrollment profile was processed but not executed because the host is enrolled into an MDM server with URL: %s", mdmServerURL) h.lastRun = time.Now().Add(-h.Frequency).Add(2 * time.Minute) - return cfg, nil + return nil } // we perform this check locally on the client too to avoid showing the @@ -104,7 +96,7 @@ func (h *renewEnrollmentProfileConfigFetcher) GetConfig() (*fleet.OrbitConfig, e // TODO: Design a better way to backoff `profiles show` so that the device doesn't get rate // limited by Apple. For now, wait at least 2 minutes before retrying. h.lastRun = time.Now().Add(-h.Frequency).Add(2 * time.Minute) - return cfg, nil + return nil } fn := h.runCmdFn @@ -125,15 +117,12 @@ func (h *renewEnrollmentProfileConfigFetcher) GetConfig() (*fleet.OrbitConfig, e } } } - return cfg, err + return nil } type execWinAPIFunc func(WindowsMDMEnrollmentArgs) error -type windowsMDMEnrollmentConfigFetcher struct { - // Fetcher is the OrbitConfigFetcher that will be wrapped. It is responsible - // for actually returning the orbit configuration or an error. - Fetcher OrbitConfigFetcher +type windowsMDMEnrollmentConfigReceiver struct { // Frequency is the minimum amount of time that must pass between two // executions of the windows MDM enrollment attempt. Frequency time.Duration @@ -161,13 +150,11 @@ type OrbitNodeKeyGetter interface { } func ApplyWindowsMDMEnrollmentFetcherMiddleware( - fetcher OrbitConfigFetcher, frequency time.Duration, hostUUID string, nodeKeyGetter OrbitNodeKeyGetter, -) OrbitConfigFetcher { - return &windowsMDMEnrollmentConfigFetcher{ - Fetcher: fetcher, +) fleet.OrbitConfigReceiver { + return &windowsMDMEnrollmentConfigReceiver{ Frequency: frequency, HostUUID: hostUUID, nodeKeyGetter: nodeKeyGetter, @@ -179,20 +166,16 @@ var errIsWindowsServer = errors.New("device is a Windows Server") // GetConfig calls the wrapped Fetcher's GetConfig method, and if the fleet // server set the "needs windows enrollment" flag to true, executes the command // to enroll into Windows MDM (or not, if the device is a Windows Server). -func (w *windowsMDMEnrollmentConfigFetcher) GetConfig() (*fleet.OrbitConfig, error) { - cfg, err := w.Fetcher.GetConfig() - - if err == nil { - if cfg.Notifications.NeedsProgrammaticWindowsMDMEnrollment { - w.attemptEnrollment(cfg.Notifications) - } else if cfg.Notifications.NeedsProgrammaticWindowsMDMUnenrollment { - w.attemptUnenrollment() - } +func (w *windowsMDMEnrollmentConfigReceiver) Run(cfg *fleet.OrbitConfig) error { + if cfg.Notifications.NeedsProgrammaticWindowsMDMEnrollment { + w.attemptEnrollment(cfg.Notifications) + } else if cfg.Notifications.NeedsProgrammaticWindowsMDMUnenrollment { + w.attemptUnenrollment() } - return cfg, err + return nil } -func (w *windowsMDMEnrollmentConfigFetcher) attemptEnrollment(notifs fleet.OrbitConfigNotifications) { +func (w *windowsMDMEnrollmentConfigReceiver) attemptEnrollment(notifs fleet.OrbitConfigNotifications) { if notifs.WindowsMDMDiscoveryEndpoint == "" { log.Info().Err(errors.New("discovery endpoint is missing")).Msg("skipping enrollment, discovery endpoint is empty") return @@ -242,7 +225,7 @@ func (w *windowsMDMEnrollmentConfigFetcher) attemptEnrollment(notifs fleet.Orbit } } -func (w *windowsMDMEnrollmentConfigFetcher) attemptUnenrollment() { +func (w *windowsMDMEnrollmentConfigReceiver) attemptUnenrollment() { if w.mu.TryLock() { defer w.mu.Unlock() @@ -279,17 +262,13 @@ func (w *windowsMDMEnrollmentConfigFetcher) attemptUnenrollment() { } } -type runScriptsConfigFetcher struct { - // Fetcher is the OrbitConfigFetcher that will be wrapped. It is responsible - // for actually returning the orbit configuration or an error. - Fetcher OrbitConfigFetcher - +type runScriptsConfigReceiver struct { // ScriptsExecutionEnabled indicates if this agent allows scripts execution. // If it doesn't, scripts are not executed, but a response is returned to the // Fleet server so it knows the agent processed the request. Note that this // should be set to the value of the --scripts-enabled command-line flag. An // additional, dynamic check is done automatically by the - // runScriptsConfigFetcher if this field is false to get the value from the + // runScriptsConfigReceiver if this field is false to get the value from the // MDM configuration profile. ScriptsExecutionEnabled bool @@ -315,10 +294,9 @@ type runScriptsConfigFetcher struct { } func ApplyRunScriptsConfigFetcherMiddleware( - fetcher OrbitConfigFetcher, scriptsEnabled bool, scriptsClient scripts.Client, -) (OrbitConfigFetcher, func() bool) { - scriptsFetcher := &runScriptsConfigFetcher{ - Fetcher: fetcher, + scriptsEnabled bool, scriptsClient scripts.Client, +) (fleet.OrbitConfigReceiver, func() bool) { + scriptsFetcher := &runScriptsConfigReceiver{ ScriptsExecutionEnabled: scriptsEnabled, ScriptsClient: scriptsClient, dynamicScriptsEnabledCheckInterval: 5 * time.Minute, @@ -328,7 +306,7 @@ func ApplyRunScriptsConfigFetcherMiddleware( return scriptsFetcher, scriptsFetcher.scriptsEnabled } -func (h *runScriptsConfigFetcher) runDynamicScriptsEnabledCheck() { +func (h *runScriptsConfigReceiver) runDynamicScriptsEnabledCheck() { getFleetdConfig := h.testGetFleetdConfig if getFleetdConfig == nil { getFleetdConfig = profiles.GetFleetdConfig @@ -366,10 +344,8 @@ func (h *runScriptsConfigFetcher) runDynamicScriptsEnabledCheck() { // GetConfig calls the wrapped Fetcher's GetConfig method, and if the fleet // server sent a list of scripts to execute, starts a goroutine to execute // them. -func (h *runScriptsConfigFetcher) GetConfig() (*fleet.OrbitConfig, error) { - cfg, err := h.Fetcher.GetConfig() - - if err == nil && len(cfg.Notifications.PendingScriptExecutionIDs) > 0 { +func (h *runScriptsConfigReceiver) Run(cfg *fleet.OrbitConfig) error { + if len(cfg.Notifications.PendingScriptExecutionIDs) > 0 { if h.mu.TryLock() { log.Debug().Msgf("received request to run scripts %v", cfg.Notifications.PendingScriptExecutionIDs) @@ -395,10 +371,10 @@ func (h *runScriptsConfigFetcher) GetConfig() (*fleet.OrbitConfig, error) { }() } } - return cfg, err + return nil } -func (h *runScriptsConfigFetcher) scriptsEnabled() bool { +func (h *runScriptsConfigReceiver) scriptsEnabled() bool { // scripts are always enabled if the agent is started with the // --scripts-enabled flag. If it is not started with this flag, then // scripts are enabled only if the mdm profile says so. @@ -428,11 +404,7 @@ type execGetEncryptionStatusFunc func() (status []bitlocker.VolumeStatus, err er // It returns an error if the process fails. type execDecryptVolumeFunc func(volumeID string) error -type windowsMDMBitlockerConfigFetcher struct { - // Fetcher is the OrbitConfigFetcher that will be wrapped. It is responsible - // for actually returning the orbit configuration or an error. - Fetcher OrbitConfigFetcher - +type windowsMDMBitlockerConfigReceiver struct { // Frequency is the minimum amount of time that must pass between two // executions of the windows MDM enrollment attempt. Frequency time.Duration @@ -460,12 +432,10 @@ type windowsMDMBitlockerConfigFetcher struct { } func ApplyWindowsMDMBitlockerFetcherMiddleware( - fetcher OrbitConfigFetcher, frequency time.Duration, encryptionResult DiskEncryptionKeySetter, -) OrbitConfigFetcher { - return &windowsMDMBitlockerConfigFetcher{ - Fetcher: fetcher, +) fleet.OrbitConfigReceiver { + return &windowsMDMBitlockerConfigReceiver{ Frequency: frequency, EncryptionResult: encryptionResult, } @@ -474,9 +444,8 @@ func ApplyWindowsMDMBitlockerFetcherMiddleware( // GetConfig calls the wrapped Fetcher's GetConfig method, and if the fleet // server set the "EnforceBitLockerEncryption" flag to true, executes the command // to attempt BitlockerEncryption (or not, if the device is a Windows Server). -func (w *windowsMDMBitlockerConfigFetcher) GetConfig() (*fleet.OrbitConfig, error) { - cfg, err := w.Fetcher.GetConfig() - if err == nil && cfg.Notifications.EnforceBitLockerEncryption { +func (w *windowsMDMBitlockerConfigReceiver) Run(cfg *fleet.OrbitConfig) error { + if cfg.Notifications.EnforceBitLockerEncryption { if w.mu.TryLock() { defer w.mu.Unlock() @@ -484,10 +453,10 @@ func (w *windowsMDMBitlockerConfigFetcher) GetConfig() (*fleet.OrbitConfig, erro } } - return cfg, err + return nil } -func (w *windowsMDMBitlockerConfigFetcher) attemptBitlockerEncryption(notifs fleet.OrbitConfigNotifications) { +func (w *windowsMDMBitlockerConfigReceiver) attemptBitlockerEncryption(notifs fleet.OrbitConfigNotifications) { if time.Since(w.lastRun) <= w.Frequency { log.Debug().Msg("skipped encryption process, last run was too recent") return @@ -561,7 +530,7 @@ func (w *windowsMDMBitlockerConfigFetcher) attemptBitlockerEncryption(notifs fle } // getEncryptionStatusForVolume retrieves the encryption status for a specific volume. -func (w *windowsMDMBitlockerConfigFetcher) getEncryptionStatusForVolume(volume string) (*bitlocker.EncryptionStatus, error) { +func (w *windowsMDMBitlockerConfigReceiver) getEncryptionStatusForVolume(volume string) (*bitlocker.EncryptionStatus, error) { fn := w.execGetEncryptionStatusFn if fn == nil { fn = bitlocker.GetEncryptionStatus @@ -582,7 +551,7 @@ func (w *windowsMDMBitlockerConfigFetcher) getEncryptionStatusForVolume(volume s // bitLockerActionInProgress determines an encryption/decription action is in // progress based on the reported status. -func (w *windowsMDMBitlockerConfigFetcher) bitLockerActionInProgress(status *bitlocker.EncryptionStatus) bool { +func (w *windowsMDMBitlockerConfigReceiver) bitLockerActionInProgress(status *bitlocker.EncryptionStatus) bool { if status == nil { return false } @@ -595,7 +564,7 @@ func (w *windowsMDMBitlockerConfigFetcher) bitLockerActionInProgress(status *bit } // performEncryption executes the encryption process. -func (w *windowsMDMBitlockerConfigFetcher) performEncryption(volume string) (string, error) { +func (w *windowsMDMBitlockerConfigReceiver) performEncryption(volume string) (string, error) { fn := w.execEncryptVolumeFn if fn == nil { fn = bitlocker.EncryptVolume @@ -609,7 +578,7 @@ func (w *windowsMDMBitlockerConfigFetcher) performEncryption(volume string) (str return recoveryKey, nil } -func (w *windowsMDMBitlockerConfigFetcher) decryptVolume(targetVolume string) error { +func (w *windowsMDMBitlockerConfigReceiver) decryptVolume(targetVolume string) error { fn := w.execDecryptVolumeFn if fn == nil { fn = bitlocker.DecryptVolume @@ -632,13 +601,13 @@ func (w *windowsMDMBitlockerConfigFetcher) decryptVolume(targetVolume string) er // encryption state. // // For more context, see issue #15916 -func (w *windowsMDMBitlockerConfigFetcher) isMisreportedDecryptionError(err *bitlocker.EncryptionError, status *bitlocker.EncryptionStatus) bool { +func (w *windowsMDMBitlockerConfigReceiver) isMisreportedDecryptionError(err *bitlocker.EncryptionError, status *bitlocker.EncryptionStatus) bool { return err.Code() == bitlocker.ErrorCodeNotDecrypted && status != nil && status.ConversionStatus == bitlocker.ConversionStatusFullyDecrypted } -func (w *windowsMDMBitlockerConfigFetcher) updateFleetServer(key string, err error) error { +func (w *windowsMDMBitlockerConfigReceiver) updateFleetServer(key string, err error) error { // Getting Bitlocker encryption operation error message if any // This is going to be sent to Fleet Server bitlockerError := "" diff --git a/orbit/pkg/update/notifications_test.go b/orbit/pkg/update/notifications_test.go index 2e5c4b12dd..cc5c46a2de 100644 --- a/orbit/pkg/update/notifications_test.go +++ b/orbit/pkg/update/notifications_test.go @@ -40,14 +40,11 @@ func TestRenewEnrollmentProfile(t *testing.T) { t.Run(c.desc, func(t *testing.T) { logBuf.Reset() - fetcher := &dummyConfigFetcher{ - cfg: &fleet.OrbitConfig{Notifications: fleet.OrbitConfigNotifications{RenewEnrollmentProfile: c.renewFlag}}, - } + testConfig := &fleet.OrbitConfig{Notifications: fleet.OrbitConfigNotifications{RenewEnrollmentProfile: c.renewFlag}} var cmdGotCalled bool var depAssignedCheckGotCalled bool - renewFetcher := &renewEnrollmentProfileConfigFetcher{ - Fetcher: fetcher, + renewReceiver := &renewEnrollmentProfileConfigReceiver{ Frequency: time.Hour, // doesn't matter for this test runCmdFn: func() error { cmdGotCalled = true @@ -62,9 +59,8 @@ func TestRenewEnrollmentProfile(t *testing.T) { }, } - cfg, err := renewFetcher.GetConfig() - require.NoError(t, err) // the dummy fetcher never returns an error - require.Equal(t, fetcher.cfg, cfg) // the renew enrollment wrapper properly returns the expected config + err := renewReceiver.Run(testConfig) + require.NoError(t, err) // the dummy receiver never returns an error require.Equal(t, c.wantCmdCalled, cmdGotCalled) require.Equal(t, c.wantCmdCalled, depAssignedCheckGotCalled) @@ -80,19 +76,16 @@ func TestRenewEnrollmentProfilePrevented(t *testing.T) { log.Logger = log.Output(&logBuf) t.Cleanup(func() { log.Logger = oldLog }) - fetcher := &dummyConfigFetcher{ - cfg: &fleet.OrbitConfig{Notifications: fleet.OrbitConfigNotifications{RenewEnrollmentProfile: true}}, - } + testConfig := &fleet.OrbitConfig{Notifications: fleet.OrbitConfigNotifications{RenewEnrollmentProfile: true}} var cmdCallCount int isEnrolled := false isAssigned := true chProceed := make(chan struct{}) - renewFetcher := &renewEnrollmentProfileConfigFetcher{ - Fetcher: fetcher, + renewReceiver := &renewEnrollmentProfileConfigReceiver{ Frequency: 2 * time.Second, // just to be safe with slow environments (CI) runCmdFn: func() error { - cmdCallCount++ // no need for sync, single-threaded call of this func is guaranteed by the fetcher's mutex + cmdCallCount++ // no need for sync, single-threaded call of this func is guaranteed by the receiver's mutex return nil }, checkEnrollmentFn: func() (bool, string, error) { @@ -108,18 +101,13 @@ func TestRenewEnrollmentProfilePrevented(t *testing.T) { }, } - assertResult := func(cfg *fleet.OrbitConfig, err error) { - require.NoError(t, err) - require.Equal(t, fetcher.cfg, cfg) - } - started := make(chan struct{}) go func() { close(started) // the first call will block in runCmdFn - cfg, err := renewFetcher.GetConfig() - assertResult(cfg, err) + err := renewReceiver.Run(testConfig) + require.NoError(t, err) }() <-started @@ -127,53 +115,53 @@ func TestRenewEnrollmentProfilePrevented(t *testing.T) { // won't call the command (won't be able to lock the mutex). However it will // still complete successfully without being blocked by the other call in // progress. - cfg, err := renewFetcher.GetConfig() - assertResult(cfg, err) + err := renewReceiver.Run(testConfig) + require.NoError(t, err) // unblock the first call close(chProceed) // this next call won't execute the command because of the frequency // restriction (it got called less than N seconds ago) - cfg, err = renewFetcher.GetConfig() - assertResult(cfg, err) + err = renewReceiver.Run(testConfig) + require.NoError(t, err) - // wait for the fetcher's frequency to pass - time.Sleep(renewFetcher.Frequency) + // wait for the receiver's frequency to pass + time.Sleep(renewReceiver.Frequency) // this call executes the command - cfg, err = renewFetcher.GetConfig() - assertResult(cfg, err) + err = renewReceiver.Run(testConfig) + require.NoError(t, err) - // wait for the fetcher's frequency to pass - time.Sleep(renewFetcher.Frequency) + // wait for the receiver's frequency to pass + time.Sleep(renewReceiver.Frequency) // this call doesn't execute the command since the host is already // enrolled isEnrolled = true - cfg, err = renewFetcher.GetConfig() - assertResult(cfg, err) + err = renewReceiver.Run(testConfig) + require.NoError(t, err) require.Equal(t, 2, cmdCallCount) // the initial call and the one after sleep - // wait for the fetcher's frequency to pass - time.Sleep(renewFetcher.Frequency) + // wait for the receiver's frequency to pass + time.Sleep(renewReceiver.Frequency) // this call doesn't execute the command since the assigned profile check fails isAssigned = false isEnrolled = false - cfg, err = renewFetcher.GetConfig() - assertResult(cfg, err) + err = renewReceiver.Run(testConfig) + require.NoError(t, err) require.Equal(t, 2, cmdCallCount) // the initial call and the one after sleep - // wait for the fetcher's frequency to pass - time.Sleep(renewFetcher.Frequency) + // wait for the receiver's frequency to pass + time.Sleep(renewReceiver.Frequency) // this next call won't execute the command because the backoff // for a failed assigned check is always 2 minutes - cfg, err = renewFetcher.GetConfig() - assertResult(cfg, err) + err = renewReceiver.Run(testConfig) + require.NoError(t, err) } type mockNodeKeyGetter struct{} @@ -219,17 +207,15 @@ func TestWindowsMDMEnrollment(t *testing.T) { unenroll = c.unenrollFlag != nil && *c.unenrollFlag isUnenroll = c.unenrollFlag != nil ) - fetcher := &dummyConfigFetcher{ - cfg: &fleet.OrbitConfig{Notifications: fleet.OrbitConfigNotifications{ - NeedsProgrammaticWindowsMDMEnrollment: enroll, - NeedsProgrammaticWindowsMDMUnenrollment: unenroll, - WindowsMDMDiscoveryEndpoint: c.discoveryURL, - }}, - } + + testConfig := &fleet.OrbitConfig{Notifications: fleet.OrbitConfigNotifications{ + NeedsProgrammaticWindowsMDMEnrollment: enroll, + NeedsProgrammaticWindowsMDMUnenrollment: unenroll, + WindowsMDMDiscoveryEndpoint: c.discoveryURL, + }} var enrollGotCalled, unenrollGotCalled bool - enrollFetcher := &windowsMDMEnrollmentConfigFetcher{ - Fetcher: fetcher, + enrollReceiver := &windowsMDMEnrollmentConfigReceiver{ Frequency: time.Hour, // doesn't matter for this test execEnrollFn: func(args WindowsMDMEnrollmentArgs) error { enrollGotCalled = true @@ -242,9 +228,8 @@ func TestWindowsMDMEnrollment(t *testing.T) { nodeKeyGetter: mockNodeKeyGetter{}, } - cfg, err := enrollFetcher.GetConfig() - require.NoError(t, err) // the dummy fetcher never returns an error - require.Equal(t, fetcher.cfg, cfg) // the enrollment wrapper properly returns the expected config + err := enrollReceiver.Run(testConfig) + require.NoError(t, err) // the dummy receiver never returns an error if isUnenroll { require.Equal(t, c.wantAPICalled, unenrollGotCalled) @@ -276,60 +261,52 @@ func TestWindowsMDMEnrollmentPrevented(t *testing.T) { } for _, cfg := range cfgs { t.Run(fmt.Sprintf("%+v", cfg), func(t *testing.T) { - baseFetcher := &dummyConfigFetcher{ - cfg: &fleet.OrbitConfig{Notifications: cfg}, - } + testConfig := &fleet.OrbitConfig{Notifications: cfg} var ( apiCallCount int apiErr error ) chProceed := make(chan struct{}) - fetcher := &windowsMDMEnrollmentConfigFetcher{ - Fetcher: baseFetcher, + receiver := &windowsMDMEnrollmentConfigReceiver{ Frequency: 2 * time.Second, // just to be safe with slow environments (CI) nodeKeyGetter: mockNodeKeyGetter{}, } if cfg.NeedsProgrammaticWindowsMDMEnrollment { - fetcher.execEnrollFn = func(args WindowsMDMEnrollmentArgs) error { + receiver.execEnrollFn = func(args WindowsMDMEnrollmentArgs) error { <-chProceed // will be unblocked only when allowed - apiCallCount++ // no need for sync, single-threaded call of this func is guaranteed by the fetcher's mutex + apiCallCount++ // no need for sync, single-threaded call of this func is guaranteed by the receiver's mutex return apiErr } - fetcher.execUnenrollFn = func(args WindowsMDMEnrollmentArgs) error { + receiver.execUnenrollFn = func(args WindowsMDMEnrollmentArgs) error { panic("should not be called") } } else { - fetcher.execUnenrollFn = func(args WindowsMDMEnrollmentArgs) error { + receiver.execUnenrollFn = func(args WindowsMDMEnrollmentArgs) error { <-chProceed // will be unblocked only when allowed - apiCallCount++ // no need for sync, single-threaded call of this func is guaranteed by the fetcher's mutex + apiCallCount++ // no need for sync, single-threaded call of this func is guaranteed by the receiver's mutex return apiErr } - fetcher.execEnrollFn = func(args WindowsMDMEnrollmentArgs) error { + receiver.execEnrollFn = func(args WindowsMDMEnrollmentArgs) error { panic("should not be called") } } - assertResult := func(cfg *fleet.OrbitConfig, err error) { - require.NoError(t, err) - require.Equal(t, baseFetcher.cfg, cfg) - } - go func() { // the first call will block in enroll/unenroll func - cfg, err := fetcher.GetConfig() - assertResult(cfg, err) + err := receiver.Run(testConfig) + require.NoError(t, err) }() - // wait a little bit to ensure the first `fetcher.GetConfig` call runs first. + // wait a little bit to ensure the first `receiver.Run` call runs first. time.Sleep(100 * time.Millisecond) // this call will happen while the first call is blocked in // enroll/unenrollfn, so it won't call the API (won't be able to lock the // mutex). However it will still complete successfully without being // blocked by the other call in progress. - cfg, err := fetcher.GetConfig() - assertResult(cfg, err) + err := receiver.Run(testConfig) + require.NoError(t, err) // unblock the first call and wait for it to complete close(chProceed) @@ -337,29 +314,29 @@ func TestWindowsMDMEnrollmentPrevented(t *testing.T) { // this next call won't execute the command because of the frequency // restriction (it got called less than N seconds ago) - cfg, err = fetcher.GetConfig() - assertResult(cfg, err) + err = receiver.Run(testConfig) + require.NoError(t, err) - // wait for the fetcher's frequency to pass - time.Sleep(fetcher.Frequency) + // wait for the receiver's frequency to pass + time.Sleep(receiver.Frequency) // this call executes the command, and it returns the Is Windows Server error apiErr = errIsWindowsServer - cfg, err = fetcher.GetConfig() - assertResult(cfg, err) + err = receiver.Run(testConfig) + require.NoError(t, err) // this next call won't execute the command (both due to frequency and the // detection of windows server) - cfg, err = fetcher.GetConfig() - assertResult(cfg, err) + err = receiver.Run(testConfig) + require.NoError(t, err) - // wait for the fetcher's frequency to pass - time.Sleep(fetcher.Frequency) + // wait for the receiver's frequency to pass + time.Sleep(receiver.Frequency) // this next call still won't execute the command (due to the detection of // windows server) - cfg, err = fetcher.GetConfig() - assertResult(cfg, err) + err = receiver.Run(testConfig) + require.NoError(t, err) require.Equal(t, 2, apiCallCount) // the initial call and the one that returned errIsWindowsServer after first sleep }) @@ -387,7 +364,7 @@ func TestRunScripts(t *testing.T) { return runFailure } - waitForRun := func(t *testing.T, r *runScriptsConfigFetcher) { + waitForRun := func(t *testing.T, r *runScriptsConfigReceiver) { var ok bool for start := time.Now(); !ok && time.Since(start) < time.Second; { ok = r.mu.TryLock() @@ -399,18 +376,15 @@ func TestRunScripts(t *testing.T) { t.Run("no pending scripts", func(t *testing.T) { t.Cleanup(func() { callsCount.Store(0); logBuf.Reset() }) - fetcher := &dummyConfigFetcher{ - cfg: &fleet.OrbitConfig{Notifications: fleet.OrbitConfigNotifications{ - PendingScriptExecutionIDs: nil, - }}, - } - runner := &runScriptsConfigFetcher{ - Fetcher: fetcher, + testConfig := &fleet.OrbitConfig{Notifications: fleet.OrbitConfigNotifications{ + PendingScriptExecutionIDs: nil, + }} + + runner := &runScriptsConfigReceiver{ runScriptsFn: mockRun, } - cfg, err := runner.GetConfig() - require.NoError(t, err) // the dummy fetcher never returns an error - require.Equal(t, fetcher.cfg, cfg) // the wrapper properly returns the expected config + err := runner.Run(testConfig) + require.NoError(t, err) // the dummy receiver never returns an error // the lock should be available because no goroutine was started require.True(t, runner.mu.TryLock()) @@ -421,18 +395,15 @@ func TestRunScripts(t *testing.T) { t.Run("pending scripts succeed", func(t *testing.T) { t.Cleanup(func() { callsCount.Store(0); logBuf.Reset() }) - fetcher := &dummyConfigFetcher{ - cfg: &fleet.OrbitConfig{Notifications: fleet.OrbitConfigNotifications{ - PendingScriptExecutionIDs: []string{"a", "b", "c"}, - }}, - } - runner := &runScriptsConfigFetcher{ - Fetcher: fetcher, + testConfig := &fleet.OrbitConfig{Notifications: fleet.OrbitConfigNotifications{ + PendingScriptExecutionIDs: []string{"a", "b", "c"}, + }} + + runner := &runScriptsConfigReceiver{ runScriptsFn: mockRun, } - cfg, err := runner.GetConfig() - require.NoError(t, err) // the dummy fetcher never returns an error - require.Equal(t, fetcher.cfg, cfg) // the wrapper properly returns the expected config + err := runner.Run(testConfig) + require.NoError(t, err) // the dummy receiver never returns an error waitForRun(t, runner) require.Equal(t, int64(1), callsCount.Load()) // all scripts executed in a single run @@ -443,21 +414,17 @@ func TestRunScripts(t *testing.T) { t.Run("pending scripts failed", func(t *testing.T) { t.Cleanup(func() { callsCount.Store(0); logBuf.Reset(); runFailure = nil }) - fetcher := &dummyConfigFetcher{ - cfg: &fleet.OrbitConfig{Notifications: fleet.OrbitConfigNotifications{ - PendingScriptExecutionIDs: []string{"a", "b", "c"}, - }}, - } + testConfig := &fleet.OrbitConfig{Notifications: fleet.OrbitConfigNotifications{ + PendingScriptExecutionIDs: []string{"a", "b", "c"}, + }} runFailure = io.ErrUnexpectedEOF - runner := &runScriptsConfigFetcher{ - Fetcher: fetcher, + runner := &runScriptsConfigReceiver{ runScriptsFn: mockRun, } - cfg, err := runner.GetConfig() - require.NoError(t, err) // the dummy fetcher never returns an error - require.Equal(t, fetcher.cfg, cfg) // the wrapper properly returns the expected config + err := runner.Run(testConfig) + require.NoError(t, err) // the dummy receiver never returns an error waitForRun(t, runner) require.Equal(t, int64(1), callsCount.Load()) // all scripts executed in a single run @@ -469,26 +436,21 @@ func TestRunScripts(t *testing.T) { t.Run("concurrent run prevented", func(t *testing.T) { t.Cleanup(func() { callsCount.Store(0); logBuf.Reset(); blockRun = nil }) - fetcher := &dummyConfigFetcher{ - cfg: &fleet.OrbitConfig{Notifications: fleet.OrbitConfigNotifications{ - PendingScriptExecutionIDs: []string{"a", "b", "c"}, - }}, - } + testConfig := &fleet.OrbitConfig{Notifications: fleet.OrbitConfigNotifications{ + PendingScriptExecutionIDs: []string{"a", "b", "c"}, + }} blockRun = make(chan struct{}) - runner := &runScriptsConfigFetcher{ - Fetcher: fetcher, + runner := &runScriptsConfigReceiver{ runScriptsFn: mockRun, } - cfg, err := runner.GetConfig() - require.NoError(t, err) // the dummy fetcher never returns an error - require.Equal(t, fetcher.cfg, cfg) // the wrapper properly returns the expected config + err := runner.Run(testConfig) + require.NoError(t, err) // the dummy receiver never returns an error // call it again, while the previous run is still running - cfg, err = runner.GetConfig() - require.NoError(t, err) // the dummy fetcher never returns an error - require.Equal(t, fetcher.cfg, cfg) // the wrapper properly returns the expected config + err = runner.Run(testConfig) + require.NoError(t, err) // the dummy receiver never returns an error // unblock the initial run close(blockRun) @@ -502,11 +464,9 @@ func TestRunScripts(t *testing.T) { t.Run("dynamic enabling of scripts", func(t *testing.T) { t.Cleanup(logBuf.Reset) - fetcher := &dummyConfigFetcher{ - cfg: &fleet.OrbitConfig{Notifications: fleet.OrbitConfigNotifications{ - PendingScriptExecutionIDs: []string{"a"}, - }}, - } + testConfig := &fleet.OrbitConfig{Notifications: fleet.OrbitConfigNotifications{ + PendingScriptExecutionIDs: []string{"a"}, + }} var ( scriptsEnabledCalls []bool @@ -515,8 +475,7 @@ func TestRunScripts(t *testing.T) { dynamicInterval = 300 * time.Millisecond ) - runner := &runScriptsConfigFetcher{ - Fetcher: fetcher, + runner := &runScriptsConfigReceiver{ ScriptsExecutionEnabled: false, runScriptsFn: func(r *scripts.Runner, s []string) error { scriptsEnabledCalls = append(scriptsEnabledCalls, r.ScriptExecutionEnabled) @@ -534,9 +493,8 @@ func TestRunScripts(t *testing.T) { runner.runDynamicScriptsEnabledCheck() // first call, scripts are disabled - cfg, err := runner.GetConfig() - require.NoError(t, err) // the dummy fetcher never returns an error - require.Equal(t, fetcher.cfg, cfg) // the wrapper properly returns the expected config + err := runner.Run(testConfig) + require.NoError(t, err) // the dummy receiver never returns an error waitForRun(t, runner) // swap scripts execution to true and wait to ensure the dynamic check @@ -545,10 +503,9 @@ func TestRunScripts(t *testing.T) { time.Sleep(dynamicInterval + 100*time.Millisecond) // second call, scripts are enabled (change exec ID to "b") - cfg.Notifications.PendingScriptExecutionIDs[0] = "b" - cfg, err = runner.GetConfig() - require.NoError(t, err) // the dummy fetcher never returns an error - require.Equal(t, fetcher.cfg, cfg) // the wrapper properly returns the expected config + testConfig.Notifications.PendingScriptExecutionIDs[0] = "b" + err = runner.Run(testConfig) + require.NoError(t, err) // the dummy receiver never returns an error waitForRun(t, runner) // swap scripts execution back to false and wait to ensure the dynamic @@ -557,10 +514,9 @@ func TestRunScripts(t *testing.T) { time.Sleep(dynamicInterval + 100*time.Millisecond) // third call, scripts are disabled (change exec ID to "c") - cfg.Notifications.PendingScriptExecutionIDs[0] = "c" - cfg, err = runner.GetConfig() - require.NoError(t, err) // the dummy fetcher never returns an error - require.Equal(t, fetcher.cfg, cfg) // the wrapper properly returns the expected config + testConfig.Notifications.PendingScriptExecutionIDs[0] = "c" + err = runner.Run(testConfig) + require.NoError(t, err) // the dummy receiver never returns an error waitForRun(t, runner) // validate the Scripts Enabled flags that were passed to the runScriptsFn @@ -600,11 +556,9 @@ func TestBitlockerOperations(t *testing.T) { decryptFnCalled = false ) - fetcher := &dummyConfigFetcher{ - cfg: &fleet.OrbitConfig{ - Notifications: fleet.OrbitConfigNotifications{ - EnforceBitLockerEncryption: shouldEncrypt, - }, + testConfig := &fleet.OrbitConfig{ + Notifications: fleet.OrbitConfigNotifications{ + EnforceBitLockerEncryption: shouldEncrypt, }, } @@ -616,10 +570,9 @@ func TestBitlockerOperations(t *testing.T) { return nil } - var enrollFetcher *windowsMDMBitlockerConfigFetcher + var enrollReceiver *windowsMDMBitlockerConfigReceiver setupTest := func() { - enrollFetcher = &windowsMDMBitlockerConfigFetcher{ - Fetcher: fetcher, + enrollReceiver = &windowsMDMBitlockerConfigReceiver{ Frequency: time.Hour, // doesn't matter for this test lastRun: time.Now().Add(-2 * time.Hour), EncryptionResult: clientMock, @@ -658,18 +611,16 @@ func TestBitlockerOperations(t *testing.T) { shouldEncrypt = true shouldFailEncryption = false shouldFailDecryption = false - cfg, err := enrollFetcher.GetConfig() - require.NoError(t, err) // the dummy fetcher never returns an error - require.Equal(t, fetcher.cfg, cfg) // the bitlocker wrapper properly returns the expected config + err := enrollReceiver.Run(testConfig) + require.NoError(t, err) // the dummy receiver never returns an error }) t.Run("bitlocker encryption is not performed", func(t *testing.T) { setupTest() shouldEncrypt = false shouldFailEncryption = false - cfg, err := enrollFetcher.GetConfig() - require.NoError(t, err) // the dummy fetcher never returns an error - require.Equal(t, fetcher.cfg, cfg) // the bitlocker wrapper properly returns the expected config + err := enrollReceiver.Run(testConfig) + require.NoError(t, err) // the dummy receiver never returns an error require.True(t, encryptFnCalled, "encryption function should have been called") require.False(t, decryptFnCalled, "decryption function should not be called") }) @@ -678,9 +629,8 @@ func TestBitlockerOperations(t *testing.T) { setupTest() shouldEncrypt = true shouldFailEncryption = true - cfg, err := enrollFetcher.GetConfig() - require.NoError(t, err) // the dummy fetcher never returns an error - require.Equal(t, fetcher.cfg, cfg) // the bitlocker wrapper properly returns the expected config + err := enrollReceiver.Run(testConfig) + require.NoError(t, err) // the dummy receiver never returns an error require.True(t, encryptFnCalled, "encryption function should have been called") require.False(t, decryptFnCalled, "decryption function should not be called") }) @@ -697,13 +647,12 @@ func TestBitlockerOperations(t *testing.T) { for _, status := range statusesToTest { t.Run(fmt.Sprintf("status %d", status), func(t *testing.T) { mockStatus := &bitlocker.EncryptionStatus{ConversionStatus: status} - enrollFetcher.execGetEncryptionStatusFn = func() ([]bitlocker.VolumeStatus, error) { + enrollReceiver.execGetEncryptionStatusFn = func() ([]bitlocker.VolumeStatus, error) { return []bitlocker.VolumeStatus{{DriveVolume: "C:", Status: mockStatus}}, nil } - cfg, err := enrollFetcher.GetConfig() + err := enrollReceiver.Run(testConfig) require.NoError(t, err) - require.Equal(t, fetcher.cfg, cfg) require.Contains(t, logBuf.String(), "skipping encryption as the disk is not available") require.False(t, encryptFnCalled, "encryption function should not be called") require.False(t, decryptFnCalled, "decryption function should not be called") @@ -715,16 +664,15 @@ func TestBitlockerOperations(t *testing.T) { t.Run("handle misreported decryption error", func(t *testing.T) { setupTest() mockStatus := &bitlocker.EncryptionStatus{ConversionStatus: bitlocker.ConversionStatusFullyDecrypted} - enrollFetcher.execGetEncryptionStatusFn = func() ([]bitlocker.VolumeStatus, error) { + enrollReceiver.execGetEncryptionStatusFn = func() ([]bitlocker.VolumeStatus, error) { return []bitlocker.VolumeStatus{{DriveVolume: "C:", Status: mockStatus}}, nil } - enrollFetcher.execEncryptVolumeFn = func(string) (string, error) { + enrollReceiver.execEncryptVolumeFn = func(string) (string, error) { return "", bitlocker.NewEncryptionError("", bitlocker.ErrorCodeNotDecrypted) } - cfg, err := enrollFetcher.GetConfig() + err := enrollReceiver.Run(testConfig) require.NoError(t, err) - require.Equal(t, fetcher.cfg, cfg) require.Contains(t, logBuf.String(), "disk encryption failed due to previous unsuccessful attempt, user action required") require.False(t, encryptFnCalled, "encryption function should not be called") require.False(t, decryptFnCalled, "decryption function should not be called") @@ -733,12 +681,11 @@ func TestBitlockerOperations(t *testing.T) { t.Run("decrypts the disk if previously encrypted", func(t *testing.T) { setupTest() mockStatus := &bitlocker.EncryptionStatus{ConversionStatus: bitlocker.ConversionStatusFullyEncrypted} - enrollFetcher.execGetEncryptionStatusFn = func() ([]bitlocker.VolumeStatus, error) { + enrollReceiver.execGetEncryptionStatusFn = func() ([]bitlocker.VolumeStatus, error) { return []bitlocker.VolumeStatus{{DriveVolume: "C:", Status: mockStatus}}, nil } - cfg, err := enrollFetcher.GetConfig() + err := enrollReceiver.Run(testConfig) require.NoError(t, err) - require.Equal(t, fetcher.cfg, cfg) require.Contains(t, logBuf.String(), "disk was previously encrypted. Attempting to decrypt it") require.False(t, clientMock.SetOrUpdateDiskEncryptionKeyInvoked) require.False(t, encryptFnCalled, "encryption function should not have been called") @@ -749,13 +696,12 @@ func TestBitlockerOperations(t *testing.T) { setupTest() shouldFailDecryption = true mockStatus := &bitlocker.EncryptionStatus{ConversionStatus: bitlocker.ConversionStatusFullyEncrypted} - enrollFetcher.execGetEncryptionStatusFn = func() ([]bitlocker.VolumeStatus, error) { + enrollReceiver.execGetEncryptionStatusFn = func() ([]bitlocker.VolumeStatus, error) { return []bitlocker.VolumeStatus{{DriveVolume: "C:", Status: mockStatus}}, nil } - cfg, err := enrollFetcher.GetConfig() + err := enrollReceiver.Run(testConfig) require.NoError(t, err) - require.Equal(t, fetcher.cfg, cfg) require.Contains(t, logBuf.String(), "disk was previously encrypted. Attempting to decrypt it") require.Contains(t, logBuf.String(), "decryption failed") require.True(t, clientMock.SetOrUpdateDiskEncryptionKeyInvoked) @@ -765,12 +711,11 @@ func TestBitlockerOperations(t *testing.T) { t.Run("encryption skipped if last run too recent", func(t *testing.T) { setupTest() - enrollFetcher.lastRun = time.Now().Add(-30 * time.Minute) - enrollFetcher.Frequency = 1 * time.Hour + enrollReceiver.lastRun = time.Now().Add(-30 * time.Minute) + enrollReceiver.Frequency = 1 * time.Hour - cfg, err := enrollFetcher.GetConfig() + err := enrollReceiver.Run(testConfig) require.NoError(t, err) - require.Equal(t, fetcher.cfg, cfg) require.Contains(t, logBuf.String(), "skipped encryption process, last run was too recent") require.False(t, encryptFnCalled, "encryption function should not be called") require.False(t, decryptFnCalled, "decryption function should not be called") @@ -780,13 +725,12 @@ func TestBitlockerOperations(t *testing.T) { setupTest() shouldFailEncryption = false mockStatus := &bitlocker.EncryptionStatus{ConversionStatus: bitlocker.ConversionStatusFullyDecrypted} - enrollFetcher.execGetEncryptionStatusFn = func() ([]bitlocker.VolumeStatus, error) { + enrollReceiver.execGetEncryptionStatusFn = func() ([]bitlocker.VolumeStatus, error) { return []bitlocker.VolumeStatus{{DriveVolume: "C:", Status: mockStatus}}, nil } - cfg, err := enrollFetcher.GetConfig() + err := enrollReceiver.Run(testConfig) require.NoError(t, err) - require.Equal(t, fetcher.cfg, cfg) require.True(t, clientMock.SetOrUpdateDiskEncryptionKeyInvoked) require.True(t, encryptFnCalled, "encryption function should have been called") require.False(t, decryptFnCalled, "decryption function should not be called") @@ -797,13 +741,12 @@ func TestBitlockerOperations(t *testing.T) { shouldFailEncryption = false shouldFailServerUpdate = true mockStatus := &bitlocker.EncryptionStatus{ConversionStatus: bitlocker.ConversionStatusFullyDecrypted} - enrollFetcher.execGetEncryptionStatusFn = func() ([]bitlocker.VolumeStatus, error) { + enrollReceiver.execGetEncryptionStatusFn = func() ([]bitlocker.VolumeStatus, error) { return []bitlocker.VolumeStatus{{DriveVolume: "C:", Status: mockStatus}}, nil } - cfg, err := enrollFetcher.GetConfig() + err := enrollReceiver.Run(testConfig) require.NoError(t, err) - require.Equal(t, fetcher.cfg, cfg) require.Contains(t, logBuf.String(), "failed to send encryption result to Fleet Server") require.True(t, clientMock.SetOrUpdateDiskEncryptionKeyInvoked) require.True(t, encryptFnCalled, "encryption function should have been called") diff --git a/orbit/pkg/update/nudge.go b/orbit/pkg/update/nudge.go index b02e075baf..f69fef327e 100644 --- a/orbit/pkg/update/nudge.go +++ b/orbit/pkg/update/nudge.go @@ -22,14 +22,11 @@ const ( nudgeConfigFileMode = os.FileMode(constant.DefaultWorldReadableFileMode) ) -// NudgeConfigFetcher is a kind of middleware that wraps an OrbitConfigFetcher and a Runner. +// NudgeConfigReceiver is a kind of middleware that wraps an OrbitConfigFetcher and a Runner. // It checks the config supplied by the wrapped OrbitConfigFetcher to detects whether the Fleet // server has supplied a Nudge config. If so, it sets Nudge as a target on the wrapped Runner. -type NudgeConfigFetcher struct { - // Fetcher is the OrbitConfigFetcher that will be wrapped. It is responsible - // for actually returning the orbit configuration or an error. - Fetcher OrbitConfigFetcher - opt NudgeConfigFetcherOptions +type NudgeConfigReceiver struct { + opt NudgeConfigFetcherOptions // ensures only one command runs at a time, protects access to lastRun cmdMu sync.Mutex lastRun time.Time @@ -53,8 +50,8 @@ type NudgeConfigFetcherOptions struct { runNudgeFn func(execPath, configPath string) error } -func ApplyNudgeConfigFetcherMiddleware(f OrbitConfigFetcher, opt NudgeConfigFetcherOptions) OrbitConfigFetcher { - return &NudgeConfigFetcher{Fetcher: f, opt: opt} +func ApplyNudgeConfigReceiverMiddleware(opt NudgeConfigFetcherOptions) fleet.OrbitConfigReceiver { + return &NudgeConfigReceiver{opt: opt} } // GetConfig calls the wrapped Fetcher's GetConfig method, and detects if the @@ -65,22 +62,17 @@ func ApplyNudgeConfigFetcherMiddleware(f OrbitConfigFetcher, opt NudgeConfigFetc // - ensures that Nudge is installed and updated via the designated TUF server. // - ensures that Nudge is opened at an interval given by n.frequency with the // provided config. -func (n *NudgeConfigFetcher) GetConfig() (*fleet.OrbitConfig, error) { +func (n *NudgeConfigReceiver) Run(cfg *fleet.OrbitConfig) error { log.Debug().Msg("running nudge config fetcher middleware") - cfg, err := n.Fetcher.GetConfig() - if err != nil { - log.Debug().Err(err).Msg("calling GetConfig from NudgeConfigFetcher") - return nil, err - } if cfg == nil { - log.Debug().Msg("NudgeConfigFetcher received nil config") - return nil, nil + log.Debug().Msg("NudgeConfigReceiver received nil config") + return nil } if n.opt.UpdateRunner == nil { - log.Debug().Msg("NudgeConfigFetcher received nil UpdateRunner, this probably indicates that updates are turned off. Skipping any actions related to Nudge") - return cfg, nil + log.Debug().Msg("NudgeConfigReceiver received nil UpdateRunner, this probably indicates that updates are turned off. Skipping any actions related to Nudge") + return nil } if cfg.NudgeConfig == nil { @@ -91,7 +83,7 @@ func (n *NudgeConfigFetcher) GetConfig() (*fleet.OrbitConfig, error) { // knowingly decided to do this as a post MVP optimization. n.opt.UpdateRunner.RemoveRunnerOptTarget("nudge") n.opt.UpdateRunner.updater.RemoveTargetInfo("nudge") - return cfg, nil + return nil } updaterHasTarget := n.opt.UpdateRunner.HasRunnerOptTarget("nudge") @@ -99,23 +91,23 @@ func (n *NudgeConfigFetcher) GetConfig() (*fleet.OrbitConfig, error) { if !updaterHasTarget || !runnerHasLocalHash { log.Info().Msg("refreshing the update runner config with Nudge targets and hashes") log.Debug().Msgf("updater has target: %t, runner has local hash: %t", updaterHasTarget, runnerHasLocalHash) - return cfg, n.setTargetsAndHashes() + return n.setTargetsAndHashes() } if err := n.configure(*cfg.NudgeConfig); err != nil { log.Info().Err(err).Msg("nudge configuration") - return cfg, err + return err } if err := n.launch(); err != nil { log.Info().Err(err).Msg("nudge launch") - return cfg, err + return err } - return cfg, nil + return nil } -func (n *NudgeConfigFetcher) setTargetsAndHashes() error { +func (n *NudgeConfigReceiver) setTargetsAndHashes() error { n.opt.UpdateRunner.AddRunnerOptTarget("nudge") n.opt.UpdateRunner.updater.SetTargetInfo("nudge", NudgeMacOSTarget) // we don't want to keep nudge as a target if we failed to update the @@ -129,7 +121,7 @@ func (n *NudgeConfigFetcher) setTargetsAndHashes() error { return nil } -func (n *NudgeConfigFetcher) configure(nudgeCfg fleet.NudgeConfig) error { +func (n *NudgeConfigReceiver) configure(nudgeCfg fleet.NudgeConfig) error { jsonCfg, err := json.Marshal(nudgeCfg) if err != nil { return err @@ -180,7 +172,7 @@ func (n *NudgeConfigFetcher) configure(nudgeCfg fleet.NudgeConfig) error { return nil } -func (n *NudgeConfigFetcher) launch() error { +func (n *NudgeConfigReceiver) launch() error { cfgFile := filepath.Join(n.opt.RootDir, nudgeConfigFile) if n.cmdMu.TryLock() { diff --git a/orbit/pkg/update/nudge_test.go b/orbit/pkg/update/nudge_test.go index b1566ba855..05d02686c1 100644 --- a/orbit/pkg/update/nudge_test.go +++ b/orbit/pkg/update/nudge_test.go @@ -38,8 +38,7 @@ func (s *nudgeTestSuite) TestUpdatesDisabled() { runNudgeFn := func(execPath, configPath string) error { return nil } - var f OrbitConfigFetcher = &dummyConfigFetcher{cfg: cfg} - f = ApplyNudgeConfigFetcherMiddleware(f, NudgeConfigFetcherOptions{ + r := ApplyNudgeConfigReceiverMiddleware(NudgeConfigFetcherOptions{ UpdateRunner: nil, RootDir: t.TempDir(), Interval: time.Minute, @@ -47,9 +46,8 @@ func (s *nudgeTestSuite) TestUpdatesDisabled() { }) // we used to get a panic if updates were disabled (see #11980) - gotCfg, err := f.GetConfig() + err = r.Run(cfg) require.NoError(t, err) - require.Equal(t, cfg, gotCfg) } func (s *nudgeTestSuite) TestNudgeConfigFetcherAddNudge() { @@ -82,8 +80,7 @@ func (s *nudgeTestSuite) TestNudgeConfigFetcherAddNudge() { return nil } - var f OrbitConfigFetcher = &dummyConfigFetcher{cfg: cfg} - f = ApplyNudgeConfigFetcherMiddleware(f, NudgeConfigFetcherOptions{ + r := ApplyNudgeConfigReceiverMiddleware(NudgeConfigFetcherOptions{ UpdateRunner: runner, RootDir: tmpDir, Interval: interval, @@ -93,9 +90,8 @@ func (s *nudgeTestSuite) TestNudgeConfigFetcherAddNudge() { // nudge is not added to targets if nudge config is not present cfg.NudgeConfig = nil - gotCfg, err := f.GetConfig() + err := r.Run(cfg) require.NoError(t, err) - require.Equal(t, cfg, gotCfg) targets := runner.updater.opt.Targets require.Len(t, targets, 0) @@ -104,9 +100,8 @@ func (s *nudgeTestSuite) TestNudgeConfigFetcherAddNudge() { require.NoError(t, err) // there's an error when the remote repo doesn't have the target yet - gotCfg, err = f.GetConfig() + err = r.Run(cfg) require.ErrorContains(t, err, "tuf: file not found") - require.Equal(t, cfg, gotCfg) // add nuge to the remote s.addRemoteTarget(nudgePath) @@ -114,9 +109,8 @@ func (s *nudgeTestSuite) TestNudgeConfigFetcherAddNudge() { // nothing happens if a nil runner is provided // nudge is added to targets when nudge config is present - gotCfg, err = f.GetConfig() + err = r.Run(cfg) require.NoError(t, err) - require.Equal(t, cfg, gotCfg) targets = runner.updater.opt.Targets require.Len(t, targets, 1) ti, ok := targets["nudge"] @@ -136,9 +130,8 @@ func (s *nudgeTestSuite) TestNudgeConfigFetcherAddNudge() { require.True(t, updated) // doesn't re-update after an update - gotCfg, err = f.GetConfig() + err = r.Run(cfg) require.NoError(t, err) - require.Equal(t, cfg, gotCfg) updated, err = runner.UpdateAction() require.NoError(t, err) require.False(t, updated) @@ -149,9 +142,8 @@ func (s *nudgeTestSuite) TestNudgeConfigFetcherAddNudge() { require.NotEmpty(t, b) // a config is created on the next run after install - gotCfg, err = f.GetConfig() + err = r.Run(cfg) require.NoError(t, err) - require.Equal(t, cfg, gotCfg) configBytes, err := os.ReadFile(configPath) require.NoError(t, err) var savedConfig fleet.NudgeConfig @@ -161,9 +153,8 @@ func (s *nudgeTestSuite) TestNudgeConfigFetcherAddNudge() { // config on disk changes if the config from the server changes cfg.NudgeConfig.OSVersionRequirements[0].RequiredMinimumOSVersion = "13.1.1" - gotCfg, err = f.GetConfig() + err = r.Run(cfg) require.NoError(t, err) - require.Equal(t, cfg, gotCfg) configBytes, err = os.ReadFile(configPath) require.NoError(t, err) savedConfig = fleet.NudgeConfig{} @@ -174,9 +165,8 @@ func (s *nudgeTestSuite) TestNudgeConfigFetcherAddNudge() { // config permissions are always validated and set to the right value err = os.Chmod(configPath, constant.DefaultFileMode) require.NoError(t, err) - gotCfg, err = f.GetConfig() + err = r.Run(cfg) require.NoError(t, err) - require.Equal(t, cfg, gotCfg) fileInfo, err := os.Stat(configPath) require.NoError(t, err) require.Equal(t, fileInfo.Mode(), nudgeConfigFileMode) @@ -203,7 +193,7 @@ func (s *nudgeTestSuite) TestNudgeConfigFetcherAddNudge() { // nudge launches successfully time.Sleep(1 * time.Second) execCmd = mockExecCommand(t, "mock stdout", "", wantCmd, wantArgs...) - _, err = f.GetConfig() + err = r.Run(cfg) require.NoError(t, err) require.Equal(t, "mock stdout", execOut) require.True(t, runNudgeFnInvoked) @@ -215,7 +205,7 @@ func (s *nudgeTestSuite) TestNudgeConfigFetcherAddNudge() { execCmd = func(command string, args ...string) *exec.Cmd { return exec.Command("non-existent-command") } - _, err = f.GetConfig() + err = r.Run(cfg) require.ErrorContains(t, err, "exec: \"non-existent-command\": executable file not found in") require.Empty(t, execOut) require.True(t, runNudgeFnInvoked) @@ -224,7 +214,7 @@ func (s *nudgeTestSuite) TestNudgeConfigFetcherAddNudge() { // nudge launches successfully time.Sleep(1 * time.Second) execCmd = mockExecCommand(t, "mock stdout", "", wantCmd, wantArgs...) - _, err = f.GetConfig() + err = r.Run(cfg) require.NoError(t, err) require.Equal(t, "mock stdout", execOut) require.True(t, runNudgeFnInvoked) @@ -234,7 +224,7 @@ func (s *nudgeTestSuite) TestNudgeConfigFetcherAddNudge() { // nudge fails to launch, stderr is captured and logged time.Sleep(1 * time.Second) execCmd = mockExecCommand(t, "", "mock stderr", wantCmd, wantArgs...) - _, err = f.GetConfig() + err = r.Run(cfg) require.ErrorContains(t, err, "exit status 1: mock stderr") require.Empty(t, execOut) require.True(t, runNudgeFnInvoked) @@ -242,17 +232,17 @@ func (s *nudgeTestSuite) TestNudgeConfigFetcherAddNudge() { // after launch error, nudge will not launch again time.Sleep(1 * time.Second) - _, err = f.GetConfig() + err = r.Run(cfg) require.NoError(t, err) require.Empty(t, execOut) require.False(t, runNudgeFnInvoked) time.Sleep(1 * time.Second) - _, err = f.GetConfig() + err = r.Run(cfg) require.NoError(t, err) require.Empty(t, execOut) require.False(t, runNudgeFnInvoked) time.Sleep(1 * time.Second) - _, err = f.GetConfig() + err = r.Run(cfg) require.NoError(t, err) require.NoError(t, err) require.Empty(t, execOut) @@ -260,9 +250,8 @@ func (s *nudgeTestSuite) TestNudgeConfigFetcherAddNudge() { // nudge is removed from targets when the config is not present cfg.NudgeConfig = nil - gotCfg, err = f.GetConfig() + err = r.Run(cfg) require.NoError(t, err) - require.Equal(t, cfg, gotCfg) targets = runner.updater.opt.Targets require.Empty(t, targets) ti, ok = targets["nudge"] diff --git a/orbit/pkg/update/swift_dialog.go b/orbit/pkg/update/swift_dialog.go index f5df2660e1..eebd68477b 100644 --- a/orbit/pkg/update/swift_dialog.go +++ b/orbit/pkg/update/swift_dialog.go @@ -6,7 +6,6 @@ import ( ) type SwiftDialogDownloader struct { - Fetcher OrbitConfigFetcher UpdateRunner *Runner } @@ -17,31 +16,26 @@ type SwiftDialogDownloaderOptions struct { } func ApplySwiftDialogDownloaderMiddleware( - f OrbitConfigFetcher, runner *Runner, -) OrbitConfigFetcher { - return &SwiftDialogDownloader{Fetcher: f, UpdateRunner: runner} +) fleet.OrbitConfigReceiver { + return &SwiftDialogDownloader{UpdateRunner: runner} } -func (s *SwiftDialogDownloader) GetConfig() (*fleet.OrbitConfig, error) { +func (s *SwiftDialogDownloader) Run(cfg *fleet.OrbitConfig) error { log.Debug().Msg("running swiftDialog installer middleware") - cfg, err := s.Fetcher.GetConfig() - if err != nil { - return nil, err - } if cfg == nil { log.Debug().Msg("SwiftDialogDownloader received nil config") - return nil, nil + return nil } if s.UpdateRunner == nil { log.Debug().Msg("SwiftDialogDownloader received nil UpdateRunner, this probably indicates that updates are turned off. Skipping any actions related to swiftDialog") - return cfg, nil + return nil } if !cfg.Notifications.NeedsMDMMigration && !cfg.Notifications.RenewEnrollmentProfile { - return cfg, nil + return nil } updaterHasTarget := s.UpdateRunner.HasRunnerOptTarget("swiftDialog") @@ -57,9 +51,9 @@ func (s *SwiftDialogDownloader) GetConfig() (*fleet.OrbitConfig, error) { log.Debug().Msgf("removing swiftDialog from target options, error updating local hashes: %s", err) s.UpdateRunner.RemoveRunnerOptTarget("swiftDialog") s.UpdateRunner.updater.RemoveTargetInfo("swiftDialog") - return cfg, err + return err } } - return cfg, nil + return nil } diff --git a/orbit/pkg/update/swift_dialog_test.go b/orbit/pkg/update/swift_dialog_test.go index 871dc2e93c..95ca2649f0 100644 --- a/orbit/pkg/update/swift_dialog_test.go +++ b/orbit/pkg/update/swift_dialog_test.go @@ -11,11 +11,9 @@ func TestSwiftDialogUpdatesDisabled(t *testing.T) { cfg := &fleet.OrbitConfig{} cfg.Notifications.NeedsMDMMigration = true cfg.Notifications.RenewEnrollmentProfile = true - var f OrbitConfigFetcher = &dummyConfigFetcher{cfg: cfg} - f = ApplySwiftDialogDownloaderMiddleware(f, nil) + r := ApplySwiftDialogDownloaderMiddleware(nil) // we used to get a panic if updates were disabled (see #11980) - gotCfg, err := f.GetConfig() + err := r.Run(cfg) require.NoError(t, err) - require.Equal(t, cfg, gotCfg) } diff --git a/server/fleet/orbit.go b/server/fleet/orbit.go index db7e73d2ad..077032d4b3 100644 --- a/server/fleet/orbit.go +++ b/server/fleet/orbit.go @@ -46,6 +46,16 @@ type OrbitConfig struct { UpdateChannels *OrbitUpdateChannels `json:"update_channels,omitempty"` } +type OrbitConfigReceiver interface { + Run(*OrbitConfig) error +} + +type OrbitConfigReceiverFunc func(cfg *OrbitConfig) error + +func (f OrbitConfigReceiverFunc) Run(cfg *OrbitConfig) error { + return f(cfg) +} + // OrbitUpdateChannels hold the update channels that can be configured in fleetd agents. type OrbitUpdateChannels struct { // Orbit holds the orbit channel. diff --git a/server/service/orbit_client.go b/server/service/orbit_client.go index 0203319b10..96323f3b99 100644 --- a/server/service/orbit_client.go +++ b/server/service/orbit_client.go @@ -2,6 +2,7 @@ package service import ( "bytes" + "context" "crypto/tls" "encoding/json" "errors" @@ -42,6 +43,16 @@ type OrbitClient struct { // TestNodeKey is used for testing only. TestNodeKey string + + // Interfaces that will receive updated configs + ConfigReceivers []fleet.OrbitConfigReceiver + // How frequently a new config will be fetched + ReceiverUpdateInterval time.Duration + // Cancelable context used by ExecuteConfigReceivers to cancel the + // update loop + ReceiverUpdateContext context.Context + // ReceiverUpdateCancelFunc will be called when ReceiverUpdateContext is cancelled + ReceiverUpdateCancelFunc context.CancelFunc } // time-to-live for config cache @@ -97,8 +108,9 @@ type OnGetConfigErrFuncs struct { } var ( - netErrInterval = 5 * time.Minute - configRetryOnNetworkError = 30 * time.Second + netErrInterval = 5 * time.Minute + configRetryOnNetworkError = 30 * time.Second + defaultOrbitConfigReceiverInterval = 30 * time.Second ) // NewOrbitClient creates a new OrbitClient. @@ -124,16 +136,87 @@ func NewOrbitClient( } nodeKeyFilePath := filepath.Join(rootDir, constant.OrbitNodeKeyFileName) + + ctx, cancelFunc := context.WithCancel(context.Background()) + return &OrbitClient{ - nodeKeyFilePath: nodeKeyFilePath, - baseClient: bc, - enrollSecret: enrollSecret, - hostInfo: orbitHostInfo, - enrolled: false, - onGetConfigErrFns: onGetConfigErrFns, + nodeKeyFilePath: nodeKeyFilePath, + baseClient: bc, + enrollSecret: enrollSecret, + hostInfo: orbitHostInfo, + enrolled: false, + onGetConfigErrFns: onGetConfigErrFns, + ReceiverUpdateInterval: defaultOrbitConfigReceiverInterval, + ReceiverUpdateContext: ctx, + ReceiverUpdateCancelFunc: cancelFunc, }, nil } +func (oc *OrbitClient) RunConfigReceivers() error { + config, err := oc.GetConfig() + if err != nil { + return fmt.Errorf("RunConfigReceivers get config: %w", err) + } + + var errs []error + var errMu sync.Mutex + var wg sync.WaitGroup + wg.Add(len(oc.ConfigReceivers)) + + for _, receiver := range oc.ConfigReceivers { + receiver := receiver + go func() { + defer func() { + if err := recover(); err != nil { + errMu.Lock() + errs = append(errs, fmt.Errorf("panic occured in receiver: %v", err)) + errMu.Unlock() + } + wg.Done() + }() + + err := receiver.Run(config) + if err != nil { + errMu.Lock() + errs = append(errs, err) + errMu.Unlock() + } + }() + } + + wg.Wait() + + if len(errs) != 0 { + return errors.Join(errs...) + } + + return nil +} + +func (oc *OrbitClient) RegisterConfigReceiver(cr fleet.OrbitConfigReceiver) { + oc.ConfigReceivers = append(oc.ConfigReceivers, cr) +} + +func (oc *OrbitClient) ExecuteConfigReceivers() error { + ticker := time.NewTicker(oc.ReceiverUpdateInterval) + defer ticker.Stop() + + for { + select { + case <-oc.ReceiverUpdateContext.Done(): + return nil + case <-ticker.C: + err := oc.RunConfigReceivers() + log.Error().Err(err) + } + } +} + +func (oc *OrbitClient) InterruptConfigReceivers(err error) { + log.Error().Err(err) + oc.ReceiverUpdateCancelFunc() +} + // GetConfig returns the Orbit config fetched from Fleet server for this instance of OrbitClient. // Since this method is called in multiple places, we use a cache with configCacheTTL time-to-live // to reduce traffic to the Fleet server. diff --git a/server/service/orbit_client_test.go b/server/service/orbit_client_test.go index f6fa57b401..377d8b0c69 100644 --- a/server/service/orbit_client_test.go +++ b/server/service/orbit_client_test.go @@ -1,11 +1,15 @@ package service import ( + "context" + "encoding/json" "errors" - "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/stretchr/testify/require" + "reflect" "testing" "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/stretchr/testify/require" ) func TestGetConfig(t *testing.T) { @@ -31,3 +35,149 @@ func TestGetConfig(t *testing.T) { }, ) } + +func clientWithConfig(cfg *fleet.OrbitConfig) *OrbitClient { + ctx, cancel := context.WithCancel(context.Background()) + oc := &OrbitClient{ + ReceiverUpdateContext: ctx, + ReceiverUpdateCancelFunc: cancel, + } + oc.configCache.config = cfg + oc.configCache.lastUpdated = time.Now().Add(1 * time.Hour) + return oc +} + +func TestConfigReceiverCalls(t *testing.T) { + var called1, called2 bool + + testmsg := json.RawMessage("testing") + + rfunc1 := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error { + if !reflect.DeepEqual(cfg.Flags, testmsg) { + return errors.New("not equal testmsg") + } + called1 = true + return nil + }) + rfunc2 := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error { + if !reflect.DeepEqual(cfg.Flags, testmsg) { + return errors.New("not equal testmsg") + } + called2 = true + return nil + }) + + client := clientWithConfig(&fleet.OrbitConfig{Flags: testmsg}) + client.RegisterConfigReceiver(rfunc1) + client.RegisterConfigReceiver(rfunc2) + + err := client.RunConfigReceivers() + require.NoError(t, err) + + require.True(t, called1) + require.True(t, called2) +} + +func TestConfigReceiverErrors(t *testing.T) { + var called1, called2 bool + + rfunc1 := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error { + called1 = true + return nil + }) + rfunc2 := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error { + called2 = true + return nil + }) + err1 := errors.New("error1") + err2 := errors.New("error2") + efunc1 := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error { + return err1 + }) + efunc2 := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error { + return err2 + }) + // Make sure we don't get stuck or crash on receiver panic + pfunc := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error { + panic("woah") + }) + + client := clientWithConfig(&fleet.OrbitConfig{}) + client.RegisterConfigReceiver(efunc1) + client.RegisterConfigReceiver(rfunc1) + client.RegisterConfigReceiver(efunc2) + client.RegisterConfigReceiver(rfunc2) + client.RegisterConfigReceiver(pfunc) + + err := client.RunConfigReceivers() + require.ErrorIs(t, err, err1) + require.ErrorIs(t, err, err2) + + require.True(t, called1) + require.True(t, called2) +} + +func TestExecuteConfigReceiversCancel(t *testing.T) { + client := clientWithConfig(&fleet.OrbitConfig{}) + client.ReceiverUpdateInterval = 100 * time.Millisecond + + var calls1, calls2 int + requiredCalls := 4 + + cfunc := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error { + calls1++ + if calls1 == requiredCalls { + client.ReceiverUpdateCancelFunc() + } + return nil + }) + + rfunc := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error { + calls2++ + return nil + }) + + client.RegisterConfigReceiver(cfunc) + client.RegisterConfigReceiver(rfunc) + + err := client.ExecuteConfigReceivers() + + require.Nil(t, err) + require.Equal(t, requiredCalls, calls1) + require.Equal(t, requiredCalls, calls2) +} + +func TestExecuteConfigReceiversInterrupt(t *testing.T) { + client := clientWithConfig(&fleet.OrbitConfig{}) + client.ReceiverUpdateInterval = 200 * time.Millisecond + + var called bool + + rfunc := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error { + called = true + return nil + }) + + client.RegisterConfigReceiver(rfunc) + + finChan := make(chan error, 1) + + go func() { + finChan <- client.ExecuteConfigReceivers() + }() + + go func() { + time.Sleep(200 * time.Millisecond) + client.ReceiverUpdateCancelFunc() + }() + + select { + case err := <-finChan: + require.Nil(t, err) + require.True(t, called) + case <-time.NewTimer(2 * time.Second).C: + require.Fail(t, "receiver interrupt cancel didn't work") + } + + client.ReceiverUpdateCancelFunc() +} From 21313fcb047742d5be868592ff13d87fea97b019 Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Thu, 9 May 2024 15:45:53 -0500 Subject: [PATCH 28/56] Update frontend to support software install activities (#18871) --- frontend/interfaces/activity.ts | 15 +- frontend/interfaces/software.ts | 29 ++++ .../cards/ActivityFeed/ActivityFeed.tsx | 16 ++ .../ActivityItem/ActivityItem.tsx | 40 +++++ .../SoftwareDetailsSummary.tsx | 4 +- .../SoftwareInstallDetails.tsx | 148 ++++++++++++++++++ .../SoftwareInstallDetails/_styles.scss | 17 ++ .../SoftwareInstallDetails/index.ts | 4 + .../HostDetailsPage/HostDetailsPage.tsx | 23 ++- .../hosts/details/cards/Activity/Activity.tsx | 6 +- .../details/cards/Activity/ActivityConfig.tsx | 10 +- .../InstalledSoftwareActivityItem.tsx | 46 ++++++ .../InstalledSoftwareActivityItem/index.ts | 1 + .../PastActivityFeed/PastActivityFeed.tsx | 8 +- .../UpcomingActivity/UpcomingActivity.tsx | 60 ++++--- .../UpcomingActivityFeed.tsx | 9 +- frontend/services/entities/activities.ts | 10 +- frontend/services/entities/software.ts | 6 + frontend/utilities/endpoints.ts | 2 + frontend/utilities/helpers.tsx | 1 + 20 files changed, 405 insertions(+), 50 deletions(-) create mode 100644 frontend/pages/SoftwarePage/components/SoftwareInstallDetails/SoftwareInstallDetails.tsx create mode 100644 frontend/pages/SoftwarePage/components/SoftwareInstallDetails/_styles.scss create mode 100644 frontend/pages/SoftwarePage/components/SoftwareInstallDetails/index.ts create mode 100644 frontend/pages/hosts/details/cards/Activity/ActivityItems/InstalledSoftwareActivityItem/InstalledSoftwareActivityItem.tsx create mode 100644 frontend/pages/hosts/details/cards/Activity/ActivityItems/InstalledSoftwareActivityItem/index.ts diff --git a/frontend/interfaces/activity.ts b/frontend/interfaces/activity.ts index a59e9c3773..d61b327676 100644 --- a/frontend/interfaces/activity.ts +++ b/frontend/interfaces/activity.ts @@ -73,13 +73,15 @@ export enum ActivityType { ResentConfigurationProfile = "resent_configuration_profile", AddedSoftware = "added_software", DeletedSoftware = "deleted_software", + InstalledSoftware = "installed_software", } // This is a subset of ActivityType that are shown only for the host past activities -export type IHostPastActivityType = +export type IHostActivityType = | ActivityType.RanScript | ActivityType.LockedHost - | ActivityType.UnlockedHost; + | ActivityType.UnlockedHost + | ActivityType.InstalledSoftware; export interface IActivity { created_at: string; @@ -92,8 +94,9 @@ export interface IActivity { details?: IActivityDetails; } -export type IPastActivity = Omit & { - type: IHostPastActivityType; +export type IHostActivity = Omit & { + type: IHostActivityType; + details: IActivityDetails; }; export interface IActivityDetails { @@ -120,6 +123,7 @@ export interface IActivityDetails { host_display_name?: string; host_display_names?: string[]; host_ids?: number[]; + host_id?: number; host_platform?: string; installed_from_dep?: boolean; mdm_platform?: "microsoft" | "apple"; @@ -134,7 +138,8 @@ export interface IActivityDetails { deadline_days?: number; grace_period_days?: number; stats?: ISchedulableQueryStats; - host_id?: number; software_title?: string; software_package?: string; + status?: string; + install_uuid?: string; } diff --git a/frontend/interfaces/software.ts b/frontend/interfaces/software.ts index 8023b25088..064d583ccc 100644 --- a/frontend/interfaces/software.ts +++ b/frontend/interfaces/software.ts @@ -148,3 +148,32 @@ export const formatSoftwareType = ({ } return type; }; +/* + * SoftwareInstallStatus represents the possible states of software install operations. + * + */ +export type SoftwareInstallStatus = "pending" | "installed" | "failed"; + +/** + * + * ISoftwareInstallResult is the shape of a software install result object + * returned by the Fleet API. + * + */ +export interface ISoftwareInstallResult { + install_uuid: string; + software_title: string; + software_title_id: number; + software_package: string; + host_id: number; + host_display_name: string; + status: SoftwareInstallStatus; + detail: string; + output: string; + pre_install_query_output: string; + post_install_script_output: string; +} + +export interface ISoftwareInstallResults { + results: ISoftwareInstallResult; +} diff --git a/frontend/pages/DashboardPage/cards/ActivityFeed/ActivityFeed.tsx b/frontend/pages/DashboardPage/cards/ActivityFeed/ActivityFeed.tsx index ed693a2a24..e4a6f90c7b 100644 --- a/frontend/pages/DashboardPage/cards/ActivityFeed/ActivityFeed.tsx +++ b/frontend/pages/DashboardPage/cards/ActivityFeed/ActivityFeed.tsx @@ -15,6 +15,9 @@ import Button from "components/buttons/Button"; import Spinner from "components/Spinner"; // @ts-ignore import FleetIcon from "components/icons/FleetIcon"; + +import { SoftwareInstallDetailsModal } from "pages/SoftwarePage/components/SoftwareInstallDetails"; + import ActivityItem from "./ActivityItem"; import ScriptDetailsModal from "./components/ScriptDetailsModal/ScriptDetailsModal"; @@ -35,6 +38,7 @@ const ActivityFeed = ({ const [pageIndex, setPageIndex] = useState(0); const [showShowQueryModal, setShowShowQueryModal] = useState(false); const [showScriptDetailsModal, setShowScriptDetailsModal] = useState(false); + const [installedSoftwareUuid, setInstalledSoftwareUuid] = useState(""); const queryShown = useRef(""); const queryImpact = useRef(undefined); const scriptExecutionId = useRef(""); @@ -81,6 +85,7 @@ const ActivityFeed = ({ activityType: ActivityType, details: IActivityDetails ) => { + console.log("activityType", activityType); switch (activityType) { case ActivityType.LiveQuery: queryShown.current = details.query_sql ?? ""; @@ -93,6 +98,11 @@ const ActivityFeed = ({ scriptExecutionId.current = details.script_execution_id ?? ""; setShowScriptDetailsModal(true); break; + case ActivityType.InstalledSoftware: + // installUuid.current = details.install_uuid ?? ""; + // console.log("installUuid.current", installUuid.current); + setInstalledSoftwareUuid(details.install_uuid ?? ""); + break; default: break; } @@ -184,6 +194,12 @@ const ActivityFeed = ({ onCancel={() => setShowScriptDetailsModal(false)} /> )} + {installedSoftwareUuid && ( + setInstalledSoftwareUuid("")} + /> + )}
); }; diff --git a/frontend/pages/DashboardPage/cards/ActivityFeed/ActivityItem/ActivityItem.tsx b/frontend/pages/DashboardPage/cards/ActivityFeed/ActivityItem/ActivityItem.tsx index 3407f2363e..7378d190ee 100644 --- a/frontend/pages/DashboardPage/cards/ActivityFeed/ActivityItem/ActivityItem.tsx +++ b/frontend/pages/DashboardPage/cards/ActivityFeed/ActivityItem/ActivityItem.tsx @@ -16,6 +16,7 @@ import Icon from "components/Icon"; import ReactTooltip from "react-tooltip"; import PremiumFeatureIconWithTooltip from "components/PremiumFeatureIconWithTooltip"; import { COLORS } from "styles/var/colors"; +import { getSoftwareInstallStatusPredicate } from "pages/hosts/details/cards/Activity/ActivityItems/InstalledSoftwareActivityItem/InstalledSoftwareActivityItem"; const baseClass = "activity-item"; @@ -854,6 +855,42 @@ const TAGGED_TEMPLATES = { ); }, + installedSoftware: ( + activity: IActivity, + onDetailsClick?: (type: ActivityType, details: IActivityDetails) => void + ) => { + const { details } = activity; + if (!details) { + return TAGGED_TEMPLATES.defaultActivityTemplate(activity); + } + + console.log("onDetailsClick", onDetailsClick); + + const { + host_display_name: hostName, + software_title: title, + status, + install_uuid, + } = details; + + return ( + <> + {" "} + {getSoftwareInstallStatusPredicate(status)} {title} software on{" "} + {hostName}.{" "} + + + ); + }, }; const getDetail = ( @@ -1033,6 +1070,9 @@ const getDetail = ( case ActivityType.DeletedSoftware: { return TAGGED_TEMPLATES.deletedSoftware(activity); } + case ActivityType.InstalledSoftware: { + return TAGGED_TEMPLATES.installedSoftware(activity, onDetailsClick); + } default: { return TAGGED_TEMPLATES.defaultActivityTemplate(activity); diff --git a/frontend/pages/SoftwarePage/components/SoftwareDetailsSummary/SoftwareDetailsSummary.tsx b/frontend/pages/SoftwarePage/components/SoftwareDetailsSummary/SoftwareDetailsSummary.tsx index f35332c84a..bf18d8c47d 100644 --- a/frontend/pages/SoftwarePage/components/SoftwareDetailsSummary/SoftwareDetailsSummary.tsx +++ b/frontend/pages/SoftwarePage/components/SoftwareDetailsSummary/SoftwareDetailsSummary.tsx @@ -41,9 +41,9 @@ const SoftwareDetailsSummary = ({

{title}

- {type && } + {!!type && } - {versions && } + {!!versions && }
diff --git a/frontend/pages/SoftwarePage/components/SoftwareInstallDetails/SoftwareInstallDetails.tsx b/frontend/pages/SoftwarePage/components/SoftwareInstallDetails/SoftwareInstallDetails.tsx new file mode 100644 index 0000000000..4c9c5fa77b --- /dev/null +++ b/frontend/pages/SoftwarePage/components/SoftwareInstallDetails/SoftwareInstallDetails.tsx @@ -0,0 +1,148 @@ +import React from "react"; +import { useQuery } from "react-query"; + +import { + ISoftwareInstallResult, + ISoftwareInstallResults, + SoftwareInstallStatus, +} from "interfaces/software"; +import softwareAPI from "services/entities/software"; + +import Modal from "components/Modal"; +import Button from "components/buttons/Button"; +import Icon from "components/Icon"; +import Textarea from "components/Textarea"; +import DataError from "components/DataError/DataError"; +import Spinner from "components/Spinner/Spinner"; +import { IconNames } from "components/icons"; + +const baseClass = "software-install-details"; + +const STATUS_ICONS: Record = { + pending: "pending-outline", + installed: "success-outline", + failed: "error-outline", +} as const; + +const STATUS_PREDICATES: Record = { + pending: "will install", + installed: "installed", + failed: "failed to install", +} as const; + +const StatusMessage = ({ + result: { host_display_name, software_package, software_title, status }, +}: { + result: ISoftwareInstallResult; +}) => { + return ( +
+ + + Fleet {STATUS_PREDICATES[status]} {software_title} ( + {software_package}) on {host_display_name} + {status === "pending" ? " when it comes online" : ""}. + +
+ ); +}; + +const OUTPUT_DISPLAY_LABELS = { + pre_install_query_output: "Pre-install condition", + output: "Software install output", + post_install_script_output: "Post-install script output", +} as const; + +const Output = ({ + displayKey, + result, +}: { + displayKey: keyof typeof OUTPUT_DISPLAY_LABELS; + result: ISoftwareInstallResult; +}) => { + return ( +
+ {OUTPUT_DISPLAY_LABELS[displayKey]}: + +
+ ); +}; + +export const SoftwareInstallDetails = ({ + installUuid, +}: { + installUuid: string; +}) => { + const { data: result, isLoading, isError } = useQuery< + ISoftwareInstallResults, + Error, + ISoftwareInstallResult + >( + ["softwareInstallResults", installUuid], + () => { + return softwareAPI.getSoftwareInstallResult(installUuid); + }, + { + refetchOnWindowFocus: false, + select: (data) => data.results, + } + ); + + if (isLoading) { + return ; + } else if (isError) { + return ; + } else if (!result) { + // FIXME: Find a better solution for this. + return ; + } + + return ( + <> +
+ + {result.status !== "pending" && ( + <> + {result.pre_install_query_output && ( + + )} + {result.output && } + {result.post_install_script_output && ( + + )} + + )} +
+ + ); +}; + +export const SoftwareInstallDetailsModal = ({ + installUuid, + onCancel, +}: { + installUuid: string; + onCancel: () => void; +}) => { + return ( + + <> +
+ +
+
+ +
+ +
+ ); +}; diff --git a/frontend/pages/SoftwarePage/components/SoftwareInstallDetails/_styles.scss b/frontend/pages/SoftwarePage/components/SoftwareInstallDetails/_styles.scss new file mode 100644 index 0000000000..390ae3a59e --- /dev/null +++ b/frontend/pages/SoftwarePage/components/SoftwareInstallDetails/_styles.scss @@ -0,0 +1,17 @@ +.software-install-details { + .modal__content { + margin-top: $pad-xlarge; + } + &__status-message { + display: flex; + align-items: center; + gap: $pad-small; + margin: 0; + } + &__script-output { + padding-top: $pad-xlarge; + .textarea { + margin-top: $pad-medium; + } + } +} diff --git a/frontend/pages/SoftwarePage/components/SoftwareInstallDetails/index.ts b/frontend/pages/SoftwarePage/components/SoftwareInstallDetails/index.ts new file mode 100644 index 0000000000..aaaabc0082 --- /dev/null +++ b/frontend/pages/SoftwarePage/components/SoftwareInstallDetails/index.ts @@ -0,0 +1,4 @@ +export { + SoftwareInstallDetails, + SoftwareInstallDetailsModal, +} from "./SoftwareInstallDetails"; diff --git a/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx b/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx index 5370d3f6b9..ebfa15770b 100644 --- a/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx +++ b/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx @@ -13,7 +13,7 @@ import { QueryContext } from "context/query"; import { NotificationContext } from "context/notification"; import activitiesAPI, { - IPastActivitiesResponse, + IHostActivitiesResponse, IUpcomingActivitiesResponse, } from "services/entities/activities"; import hostAPI from "services/entities/hosts"; @@ -58,6 +58,7 @@ import TabsWrapper from "components/TabsWrapper"; import MainContent from "components/MainContent"; import BackLink from "components/BackLink"; import ScriptDetailsModal from "pages/DashboardPage/cards/ActivityFeed/components/ScriptDetailsModal"; +import { SoftwareInstallDetailsModal } from "pages/SoftwarePage/components/SoftwareInstallDetails"; import HostSummaryCard from "../cards/HostSummary"; import AboutCard from "../cards/About"; @@ -171,6 +172,7 @@ const HostDetailsPage = ({ const [selectedPolicy, setSelectedPolicy] = useState( null ); + const [softwareInstallUuid, setSoftwareInstallUuid] = useState(""); const [isUpdatingHost, setIsUpdatingHost] = useState(false); const [refetchStartTime, setRefetchStartTime] = useState(null); const [showRefetchSpinner, setShowRefetchSpinner] = useState(false); @@ -369,9 +371,9 @@ const HostDetailsPage = ({ isError: pastActivitiesIsError, refetch: refetchPastActivities, } = useQuery< - IPastActivitiesResponse, + IHostActivitiesResponse, Error, - IPastActivitiesResponse, + IHostActivitiesResponse, Array<{ scope: string; pageIndex: number; @@ -552,6 +554,9 @@ const HostDetailsPage = ({ case "ran_script": setScriptDetailsId(details?.script_execution_id || ""); break; + case "installed_software": + setSoftwareInstallUuid(details?.install_uuid || ""); + break; default: // do nothing } }, @@ -582,7 +587,11 @@ const HostDetailsPage = ({ const onCancelScriptDetailsModal = useCallback(() => { setScriptDetailsId(""); - }, [setScriptDetailsId]); + }, []); + + const onCancelSoftwareInstallDetailsModal = useCallback(() => { + setSoftwareInstallUuid(""); + }, []); const onTransferHostSubmit = async (team: ITeam) => { setIsUpdatingHost(true); @@ -967,6 +976,12 @@ const HostDetailsPage = ({ onCancel={onCancelScriptDetailsModal} /> )} + {!!softwareInstallUuid && ( + + )} {showLockHostModal && ( { interface IActivityProps { activeTab: "past" | "upcoming"; - activities?: IPastActivitiesResponse | IUpcomingActivitiesResponse; + activities?: IHostActivitiesResponse | IUpcomingActivitiesResponse; isLoading?: boolean; isError?: boolean; upcomingCount: number; @@ -101,7 +101,7 @@ const Activity = ({ | React.FC > = { [ActivityType.RanScript]: RanScriptActivityItem, [ActivityType.LockedHost]: LockedHostActivityItem, [ActivityType.UnlockedHost]: UnlockedHostActivityItem, + [ActivityType.InstalledSoftware]: InstalledSoftwareActivityItem, }; diff --git a/frontend/pages/hosts/details/cards/Activity/ActivityItems/InstalledSoftwareActivityItem/InstalledSoftwareActivityItem.tsx b/frontend/pages/hosts/details/cards/Activity/ActivityItems/InstalledSoftwareActivityItem/InstalledSoftwareActivityItem.tsx new file mode 100644 index 0000000000..fd0d7d3fa6 --- /dev/null +++ b/frontend/pages/hosts/details/cards/Activity/ActivityItems/InstalledSoftwareActivityItem/InstalledSoftwareActivityItem.tsx @@ -0,0 +1,46 @@ +import React from "react"; + +import { SoftwareInstallStatus } from "interfaces/software"; + +import { IHostActivityItemComponentPropsWithShowDetails } from "../../ActivityConfig"; +import HostActivityItem from "../../HostActivityItem"; +import ShowDetailsButton from "../../ShowDetailsButton"; + +const baseClass = "installed-software-activity-item"; + +const STATUS_PREDICATES: Record = { + failed: "failed to install", + installed: "installed", + pending: "told Fleet to install", +} as const; + +export const getSoftwareInstallStatusPredicate = ( + status: string | undefined +) => { + if (!status) { + return STATUS_PREDICATES.pending; + } + return ( + STATUS_PREDICATES[status as SoftwareInstallStatus] || + STATUS_PREDICATES.pending + ); +}; + +const InstalledSoftwareActivityItem = ({ + activity, + onShowDetails, +}: IHostActivityItemComponentPropsWithShowDetails) => { + const { actor_full_name: actorName, details } = activity; + + const { status, software_title: title } = details; + + return ( + + {actorName} {getSoftwareInstallStatusPredicate(status)}{" "} + {title} software on this host.{" "} + + + ); +}; + +export default InstalledSoftwareActivityItem; diff --git a/frontend/pages/hosts/details/cards/Activity/ActivityItems/InstalledSoftwareActivityItem/index.ts b/frontend/pages/hosts/details/cards/Activity/ActivityItems/InstalledSoftwareActivityItem/index.ts new file mode 100644 index 0000000000..868ba94b63 --- /dev/null +++ b/frontend/pages/hosts/details/cards/Activity/ActivityItems/InstalledSoftwareActivityItem/index.ts @@ -0,0 +1 @@ +export { default } from "./InstalledSoftwareActivityItem"; diff --git a/frontend/pages/hosts/details/cards/Activity/PastActivityFeed/PastActivityFeed.tsx b/frontend/pages/hosts/details/cards/Activity/PastActivityFeed/PastActivityFeed.tsx index 05c7a790a8..97f0346141 100644 --- a/frontend/pages/hosts/details/cards/Activity/PastActivityFeed/PastActivityFeed.tsx +++ b/frontend/pages/hosts/details/cards/Activity/PastActivityFeed/PastActivityFeed.tsx @@ -1,7 +1,7 @@ import React from "react"; -import { IPastActivity } from "interfaces/activity"; -import { IPastActivitiesResponse } from "services/entities/activities"; +import { IHostActivity } from "interfaces/activity"; +import { IHostActivitiesResponse } from "services/entities/activities"; // @ts-ignore import FleetIcon from "components/icons/FleetIcon"; @@ -16,7 +16,7 @@ import { pastActivityComponentMap } from "../ActivityConfig"; const baseClass = "past-activity-feed"; interface IPastActivityFeedProps { - activities?: IPastActivitiesResponse; + activities?: IHostActivitiesResponse; isError?: boolean; onDetailsClick: ShowActivityDetailsHandler; onNextPage: () => void; @@ -53,7 +53,7 @@ const PastActivityFeed = ({ return (
- {activitiesList.map((activity: IPastActivity) => { + {activitiesList.map((activity: IHostActivity) => { const ActivityItemComponent = pastActivityComponentMap[activity.type]; return ( { + switch (type) { + case ActivityType.RanScript: + return ( + <> + told Fleet to run{" "} + {formatScriptNameForActivityItem(details?.script_name)} + + ); + case ActivityType.InstalledSoftware: + return ( + <> + told Fleet to install{" "} + {details?.software_title ? ( + <> + {details.software_title}{" "} + + ) : ( + "" + )} + software + + ); + default: + // this should never happen + return <>{type}; + } +}; + // TODO: Combine this with ./UpcomingActivity/UpcomingActivity.tsx and // frontend/pages/DashboardPage/cards/ActivityFeed/ActivityItem/ActivityItem.tsx const UpcomingActivity = ({ @@ -46,23 +75,16 @@ const UpcomingActivity = ({
- {activity.actor_full_name} - <> - {" "} - told Fleet to run{" "} - {formatScriptNameForActivityItem( - activity.details?.script_name - )}{" "} - on this host.{" "} - - + {activity.actor_full_name} {formatPredicate(activity)} on + this host.{" "} +
void; @@ -52,8 +52,9 @@ const UpcomingActivityFeed = ({ return (
- {activitiesList.map((activity: IActivity) => ( + {activitiesList.map((activity: IHostActivity) => ( diff --git a/frontend/services/entities/activities.ts b/frontend/services/entities/activities.ts index 7a45e52b83..61123bddc4 100644 --- a/frontend/services/entities/activities.ts +++ b/frontend/services/entities/activities.ts @@ -1,5 +1,5 @@ import endpoints from "utilities/endpoints"; -import { IActivity, IPastActivity } from "interfaces/activity"; +import { IActivity, IHostActivity } from "interfaces/activity"; import sendRequest from "services"; import { buildQueryStringFromParams } from "utilities/url"; @@ -16,15 +16,15 @@ export interface IActivitiesResponse { }; } -export interface IPastActivitiesResponse { - activities: IPastActivity[] | null; +export interface IHostActivitiesResponse { + activities: IHostActivity[] | null; meta: { has_next_results: boolean; has_previous_results: boolean; }; } -export interface IUpcomingActivitiesResponse extends IActivitiesResponse { +export interface IUpcomingActivitiesResponse extends IHostActivitiesResponse { count: number; } @@ -53,7 +53,7 @@ export default { id: number, page = DEFAULT_PAGE, perPage = DEFAULT_PAGE_SIZE - ): Promise => { + ): Promise => { const { HOST_PAST_ACTIVITIES } = endpoints; const queryParams = { diff --git a/frontend/services/entities/software.ts b/frontend/services/entities/software.ts index d9ebf6bc4c..f27bb56502 100644 --- a/frontend/services/entities/software.ts +++ b/frontend/services/entities/software.ts @@ -231,4 +231,10 @@ export default { })}`; return sendRequest("GET", path); }, + + getSoftwareInstallResult: (installUuid: string) => { + const { SOFTWARE_INSTALL_RESULTS } = endpoints; + const path = SOFTWARE_INSTALL_RESULTS(installUuid); + return sendRequest("GET", path); + }, }; diff --git a/frontend/utilities/endpoints.ts b/frontend/utilities/endpoints.ts index 2245a69d15..1ecc1c726a 100644 --- a/frontend/utilities/endpoints.ts +++ b/frontend/utilities/endpoints.ts @@ -131,6 +131,8 @@ export default { SOFTWARE_PACKAGE_ADD: `/${API_VERSION}/fleet/software/package`, SOFTWARE_PACKAGE: (id: number) => `/${API_VERSION}/fleet/software/packages/${id}`, + SOFTWARE_INSTALL_RESULTS: (uuid: string) => + `/${API_VERSION}/fleet/software/install/results/${uuid}`, // AI endpoints AUTOFILL_POLICY: `/${API_VERSION}/fleet/autofill/policy`, diff --git a/frontend/utilities/helpers.tsx b/frontend/utilities/helpers.tsx index 906f249ab0..5ac8d2355c 100644 --- a/frontend/utilities/helpers.tsx +++ b/frontend/utilities/helpers.tsx @@ -55,6 +55,7 @@ import { } from "utilities/constants"; import { ISchedulableQueryStats } from "interfaces/schedulable_query"; import { IDropdownOption } from "interfaces/dropdownOption"; +import { IActivityDetails } from "interfaces/activity"; const ORG_INFO_ATTRS = ["org_name", "org_logo_url"]; const ADMIN_ATTRS = ["email", "name", "password", "password_confirmation"]; From fc1c903f63b721f912af086d3d520d12189b48bc Mon Sep 17 00:00:00 2001 From: Gabriel Hernandez Date: Thu, 9 May 2024 22:44:50 +0100 Subject: [PATCH 29/56] add UI on host details and my device page for installing software feature (#18878) relates to #18327 This adds the ui changes for installing software feature on the host details and my device page. this includes: **new software tables for host details and my device pages:** image **new actions dropdown to install and see details:** image **new software details modal:** image **use new API for showing host software in the UI:** - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [x] Manual QA for all new/changed functionality --- frontend/__mocks__/hostMock.ts | 49 +++ .../DataTable/LinkCell/_styles.scss | 0 .../SoftwareNameCell/SoftwareNameCell.tsx | 55 +++ .../DataTable/SoftwareNameCell/_styles.scss | 16 + .../DataTable/SoftwareNameCell/index.ts | 1 + frontend/interfaces/software.ts | 33 +- .../SoftwarePackageCard.tsx | 11 +- .../SoftwareTable/SoftwareTable.tsx | 3 +- .../SoftwareTitlesTableConfig.tsx | 43 +- .../SoftwareVersionsTableConfig.tsx | 23 +- .../SoftwareTitles/SoftwareTable/_styles.scss | 6 - frontend/pages/SoftwarePage/_styles.scss | 9 - .../SoftwareInstallDetails.tsx | 6 +- .../components/VersionCell/VersionCell.tsx | 18 +- .../details/DeviceUserPage/DeviceUserPage.tsx | 10 +- .../HostDetailsPage/HostDetailsPage.tsx | 34 +- .../InstalledSoftwareActivityItem.tsx | 6 +- .../Software/DeviceSoftwareTableConfig.tsx | 90 ++++ .../HostSoftwareTable/HostSoftwareTable.tsx | 124 ++++++ .../cards/Software/HostSoftwareTable/index.ts | 1 + .../Software/HostSoftwareTableConfig.tsx | 174 ++++++++ .../InstallStatusCell/InstallStatusCell.tsx | 92 ++++ .../Software/InstallStatusCell/_styles.scss | 7 + .../cards/Software/InstallStatusCell/index.ts | 1 + .../hosts/details/cards/Software/Software.tsx | 312 +++++-------- .../SoftwareDetailsModal.tsx | 126 ++++++ .../SoftwareDetailsModal/_styles.scss | 12 + .../Software/SoftwareDetailsModal/index.ts | 1 + .../cards/Software/SoftwareTableConfig.tsx | 416 ------------------ .../SoftwareVulnCount/SoftwareVulnCount.tsx | 41 -- .../Software/SoftwareVulnCount/_styles.scss | 11 - .../cards/Software/SoftwareVulnCount/index.ts | 1 - frontend/services/entities/device_user.ts | 30 ++ frontend/services/entities/hosts.ts | 37 +- frontend/services/entities/software.ts | 2 - frontend/utilities/date_format/index.ts | 4 + frontend/utilities/endpoints.ts | 26 +- 37 files changed, 1046 insertions(+), 785 deletions(-) create mode 100644 frontend/components/TableContainer/DataTable/LinkCell/_styles.scss create mode 100644 frontend/components/TableContainer/DataTable/SoftwareNameCell/SoftwareNameCell.tsx create mode 100644 frontend/components/TableContainer/DataTable/SoftwareNameCell/_styles.scss create mode 100644 frontend/components/TableContainer/DataTable/SoftwareNameCell/index.ts create mode 100644 frontend/pages/hosts/details/cards/Software/DeviceSoftwareTableConfig.tsx create mode 100644 frontend/pages/hosts/details/cards/Software/HostSoftwareTable/HostSoftwareTable.tsx create mode 100644 frontend/pages/hosts/details/cards/Software/HostSoftwareTable/index.ts create mode 100644 frontend/pages/hosts/details/cards/Software/HostSoftwareTableConfig.tsx create mode 100644 frontend/pages/hosts/details/cards/Software/InstallStatusCell/InstallStatusCell.tsx create mode 100644 frontend/pages/hosts/details/cards/Software/InstallStatusCell/_styles.scss create mode 100644 frontend/pages/hosts/details/cards/Software/InstallStatusCell/index.ts create mode 100644 frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/SoftwareDetailsModal.tsx create mode 100644 frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/_styles.scss create mode 100644 frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/index.ts delete mode 100644 frontend/pages/hosts/details/cards/Software/SoftwareTableConfig.tsx delete mode 100644 frontend/pages/hosts/details/cards/Software/SoftwareVulnCount/SoftwareVulnCount.tsx delete mode 100644 frontend/pages/hosts/details/cards/Software/SoftwareVulnCount/_styles.scss delete mode 100644 frontend/pages/hosts/details/cards/Software/SoftwareVulnCount/index.ts diff --git a/frontend/__mocks__/hostMock.ts b/frontend/__mocks__/hostMock.ts index 37b5f826bd..6800356227 100644 --- a/frontend/__mocks__/hostMock.ts +++ b/frontend/__mocks__/hostMock.ts @@ -4,6 +4,8 @@ import { pick } from "lodash"; import { normalizeEmptyValues } from "utilities/helpers"; import { HOST_SUMMARY_DATA } from "utilities/constants"; +import { IGetHostSoftwareResponse } from "services/entities/hosts"; +import { IHostSoftware } from "interfaces/software"; const DEFAULT_HOST_PROFILE_MOCK: IHostMdmProfile = { profile_uuid: "123-abc", @@ -116,4 +118,51 @@ export const createMockHostSummary = (overrides?: Partial) => { ); }; +const DEFAULT_HOST_SOFTWARE_MOCK: IHostSoftware = { + id: 1, + name: "mock software.app", + package_available_for_install: "mockSoftware.app", + source: "apps", + bundle_identifier: "com.test.mock", + status: "installed", + last_install: { + install_uuid: "123-abc", + installed_at: "2022-01-01T12:00:00Z", + }, + installed_versions: [ + { + version: "1.0.0", + last_opened_at: "2022-01-01T12:00:00Z", + vulnerabilities: ["CVE-2020-0001"], + installed_paths: ["/Applications/mock.app"], + }, + ], +}; + +export const createMockHostSoftware = ( + overrides?: Partial +): IHostSoftware => { + return { + ...DEFAULT_HOST_SOFTWARE_MOCK, + ...overrides, + }; +}; + +const DEFAULT_GET_HOST_SOFTWARE_RESPONSE_MOCK: IGetHostSoftwareResponse = { + software: [createMockHostSoftware()], + meta: { + has_next_results: false, + has_previous_results: false, + }, +}; + +export const createMockGetHostSoftwareResponse = ( + overrides?: Partial +): IGetHostSoftwareResponse => { + return { + ...DEFAULT_GET_HOST_SOFTWARE_RESPONSE_MOCK, + ...overrides, + }; +}; + export default createMockHost; diff --git a/frontend/components/TableContainer/DataTable/LinkCell/_styles.scss b/frontend/components/TableContainer/DataTable/LinkCell/_styles.scss new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frontend/components/TableContainer/DataTable/SoftwareNameCell/SoftwareNameCell.tsx b/frontend/components/TableContainer/DataTable/SoftwareNameCell/SoftwareNameCell.tsx new file mode 100644 index 0000000000..79e25e8890 --- /dev/null +++ b/frontend/components/TableContainer/DataTable/SoftwareNameCell/SoftwareNameCell.tsx @@ -0,0 +1,55 @@ +import React from "react"; +import { InjectedRouter } from "react-router"; + +import SoftwareIcon from "pages/SoftwarePage/components/icons/SoftwareIcon"; + +import LinkCell from "../LinkCell"; + +const baseClass = "software-name-cell"; + +interface ISoftwareNameCellProps { + name: string; + source: string; + path?: string; + router?: InjectedRouter; +} + +const SoftwareNameCell = ({ + name, + source, + path, + router, +}: ISoftwareNameCellProps) => { + // NO path or router means it's not clickable. return + // a non-clickable cell early + if (!router || !path) { + return ( +
+ + {name} +
+ ); + } + + const onClickSoftware = (e: React.MouseEvent) => { + // Allows for button to be clickable in a clickable row + e.stopPropagation(); + router.push(path); + }; + + return ( + + + {name} + + } + /> + ); +}; + +export default SoftwareNameCell; diff --git a/frontend/components/TableContainer/DataTable/SoftwareNameCell/_styles.scss b/frontend/components/TableContainer/DataTable/SoftwareNameCell/_styles.scss new file mode 100644 index 0000000000..a94f994132 --- /dev/null +++ b/frontend/components/TableContainer/DataTable/SoftwareNameCell/_styles.scss @@ -0,0 +1,16 @@ +.software-name-cell { + // TODO: we do not want to use !important but have to for now. We need to pull + // the .link-cell styles into the LinkCell component in order to + // decrease the specificity of the styles. This will allow us to remove the + // !important from here. + display: flex !important; + align-items: center; + gap: $pad-small; + + .software-icon { + width: 24px; + height: 24px; + border: 1px solid $ui-fleet-black-10; + border-radius: 8px; + } +} diff --git a/frontend/components/TableContainer/DataTable/SoftwareNameCell/index.ts b/frontend/components/TableContainer/DataTable/SoftwareNameCell/index.ts new file mode 100644 index 0000000000..f87c332123 --- /dev/null +++ b/frontend/components/TableContainer/DataTable/SoftwareNameCell/index.ts @@ -0,0 +1 @@ +export { default } from "./SoftwareNameCell"; diff --git a/frontend/interfaces/software.ts b/frontend/interfaces/software.ts index 064d583ccc..f6a75eac85 100644 --- a/frontend/interfaces/software.ts +++ b/frontend/interfaces/software.ts @@ -138,7 +138,7 @@ export const formatSoftwareType = ({ browser, }: { source: string; - browser: string; + browser?: string; }) => { let type = SOURCE_TYPE_CONVERSION[source] || "Unknown"; if (browser) { @@ -152,7 +152,7 @@ export const formatSoftwareType = ({ * SoftwareInstallStatus represents the possible states of software install operations. * */ -export type SoftwareInstallStatus = "pending" | "installed" | "failed"; +export type ISoftwareInstallStatus = "pending" | "installed" | "failed"; /** * @@ -167,7 +167,7 @@ export interface ISoftwareInstallResult { software_package: string; host_id: number; host_display_name: string; - status: SoftwareInstallStatus; + status: ISoftwareInstallStatus; detail: string; output: string; pre_install_query_output: string; @@ -177,3 +177,30 @@ export interface ISoftwareInstallResult { export interface ISoftwareInstallResults { results: ISoftwareInstallResult; } + +// ISoftwareInstallerType defines the supported installer types for +// software uploaded by the IT admin. +export type ISoftwareInstallerType = "pkg" | "msi" | "deb" | "exe"; + +export interface ISoftwareLastInstall { + install_uuid: string; + installed_at: string; +} + +export interface ISoftwareInstallVersion { + version: string; + last_opened_at: string | null; + vulnerabilities: string[]; + installed_paths: string[]; +} + +export interface IHostSoftware { + id: number; + name: string; + package_available_for_install: string | null; + source: string; + bundle_identifier: string; + status: ISoftwareInstallStatus | null; + last_install: ISoftwareLastInstall | null; + installed_versions: ISoftwareInstallVersion[] | null; +} diff --git a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx index e700b071c4..d0548e8563 100644 --- a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx +++ b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx @@ -1,7 +1,7 @@ import React, { useContext, useState } from "react"; import endpoints from "utilities/endpoints"; -import software, { ISoftwarePackage } from "interfaces/software"; +import { ISoftwareInstallStatus, ISoftwarePackage } from "interfaces/software"; import PATHS from "router/paths"; import { AppContext } from "context/app"; import { buildQueryStringFromParams } from "utilities/url"; @@ -20,7 +20,6 @@ import AdvancedOptionsModal from "../AdvancedOptionsModal"; const baseClass = "software-package-card"; -type IPackageInstallStatus = "installed" | "pending" | "failed"; interface IStatusDisplayOption { displayName: string; iconName: "success" | "pending-outline" | "error"; @@ -28,7 +27,7 @@ interface IStatusDisplayOption { } const STATUS_DISPLAY_OPTIONS: Record< - IPackageInstallStatus, + ISoftwareInstallStatus, IStatusDisplayOption > = { installed: { @@ -50,7 +49,7 @@ const STATUS_DISPLAY_OPTIONS: Record< interface IPackageStatusCountProps { softwareId: number; - status: IPackageInstallStatus; + status: ISoftwareInstallStatus; count: number; teamId?: number; } @@ -117,10 +116,6 @@ const SoftwarePackageCard = ({ setShowAdvancedOptionsModal(true); }; - const onDownloadClick = () => { - console.log("Download clicked"); - }; - const onDeleteClick = () => { setShowDeleteModal(true); }; diff --git a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTable.tsx b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTable.tsx index a772f0b344..fa52e1d724 100644 --- a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTable.tsx +++ b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTable.tsx @@ -3,12 +3,11 @@ software/titles Software tab > Table software/versions Software tab > Table (version toggle on) */ -import React, { useCallback, useContext, useMemo } from "react"; +import React, { useCallback, useMemo } from "react"; import { InjectedRouter } from "react-router"; import { Row } from "react-table"; import PATHS from "router/paths"; -import { AppContext } from "context/app"; import { getNextLocationPath } from "utilities/helpers"; import { GITHUB_NEW_ISSUE_LINK } from "utilities/constants"; import { buildQueryStringFromParams } from "utilities/url"; diff --git a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTitlesTableConfig.tsx b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTitlesTableConfig.tsx index e0f9cb54c2..e4626c4d95 100644 --- a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTitlesTableConfig.tsx +++ b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTitlesTableConfig.tsx @@ -2,11 +2,7 @@ import React from "react"; import { CellProps, Column } from "react-table"; import { InjectedRouter } from "react-router"; -import { - ISoftwareTitleVersion, - ISoftwareTitle, - formatSoftwareType, -} from "interfaces/software"; +import { ISoftwareTitle, formatSoftwareType } from "interfaces/software"; import PATHS from "router/paths"; import { buildQueryStringFromParams } from "utilities/url"; @@ -14,13 +10,13 @@ import { IHeaderProps, IStringCellProps } from "interfaces/datatable_config"; import HeaderCell from "components/TableContainer/DataTable/HeaderCell"; import TextCell from "components/TableContainer/DataTable/TextCell"; -import LinkCell from "components/TableContainer/DataTable/LinkCell/LinkCell"; import ViewAllHostsLink from "components/ViewAllHostsLink"; +import SoftwareNameCell from "components/TableContainer/DataTable/SoftwareNameCell"; + import IconCell from "pages/SoftwarePage/components/IconCell"; import VersionCell from "../../components/VersionCell"; import VulnerabilitiesCell from "../../components/VulnerabilitiesCell"; -import SoftwareIcon from "../../components/icons/SoftwareIcon"; // NOTE: cellProps come from react-table // more info here https://react-table.tanstack.com/docs/api/useTable#cell-properties @@ -41,7 +37,11 @@ type IViewAllHostsLinkProps = CellProps; type ITableHeaderProps = IHeaderProps; -const getVulnerabilities = (versions: ISoftwareTitleVersion[]) => { +export const getVulnerabilities = < + T extends { vulnerabilities: string[] | null } +>( + versions: T[] +) => { if (!versions) { return []; } @@ -76,36 +76,17 @@ const generateTableHeaders = ( id.toString() )}?${teamQueryParam}`; - const onClickSoftware = (e: React.MouseEvent) => { - // Allows for button to be clickable in a clickable row - e.stopPropagation(); - - router?.push(softwareTitleDetailsPath); - }; - return ( - - - {name} - - } + router={router} /> ); }, sortType: "caseInsensitive", }, - { - Header: "Install status", - disableSortBy: true, - accessor: "software_package", - Cell: (cellProps: ISoftwarePackageCellProps) => { - return cellProps.cell.value ? : null; - }, - }, { Header: "Type", disableSortBy: true, diff --git a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareVersionsTableConfig.tsx b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareVersionsTableConfig.tsx index fb8fde36d9..d22aab9e86 100644 --- a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareVersionsTableConfig.tsx +++ b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareVersionsTableConfig.tsx @@ -13,10 +13,10 @@ import PATHS from "router/paths"; import HeaderCell from "components/TableContainer/DataTable/HeaderCell"; import TextCell from "components/TableContainer/DataTable/TextCell"; -import LinkCell from "components/TableContainer/DataTable/LinkCell/LinkCell"; import ViewAllHostsLink from "components/ViewAllHostsLink"; +import SoftwareNameCell from "components/TableContainer/DataTable/SoftwareNameCell"; + import VulnerabilitiesCell from "../../components/VulnerabilitiesCell"; -import SoftwareIcon from "../../components/icons/SoftwareIcon"; // NOTE: cellProps come from react-table // more info here https://react-table.tanstack.com/docs/api/useTable#cell-properties @@ -52,23 +52,12 @@ const generateTableHeaders = ( id.toString() )}?${teamQueryParam}`; - const onClickSoftware = (e: React.MouseEvent) => { - // Allows for button to be clickable in a clickable row - e.stopPropagation(); - - router?.push(softwareVersionDetailsPath); - }; - return ( - - - {name} - - } + router={router} /> ); }, diff --git a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/_styles.scss b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/_styles.scss index 638f56d6b9..d6e62005a9 100644 --- a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/_styles.scss +++ b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/_styles.scss @@ -124,12 +124,6 @@ } } - .link-cell { - display: flex; - align-items: center; - gap: $pad-small; - } - .hosts_count__cell { .hosts-cell__wrapper { display: flex; diff --git a/frontend/pages/SoftwarePage/_styles.scss b/frontend/pages/SoftwarePage/_styles.scss index 2740a08de4..aced0a5cb3 100644 --- a/frontend/pages/SoftwarePage/_styles.scss +++ b/frontend/pages/SoftwarePage/_styles.scss @@ -67,14 +67,5 @@ .component__tabs-wrapper { margin-bottom: $pad-xxlarge; } - - .table-container { - .software-icon { - width: 24px; - height: 24px; - border: 1px solid $ui-fleet-black-10; - border-radius: 8px; - } - } } } diff --git a/frontend/pages/SoftwarePage/components/SoftwareInstallDetails/SoftwareInstallDetails.tsx b/frontend/pages/SoftwarePage/components/SoftwareInstallDetails/SoftwareInstallDetails.tsx index 4c9c5fa77b..87c2b377da 100644 --- a/frontend/pages/SoftwarePage/components/SoftwareInstallDetails/SoftwareInstallDetails.tsx +++ b/frontend/pages/SoftwarePage/components/SoftwareInstallDetails/SoftwareInstallDetails.tsx @@ -4,7 +4,7 @@ import { useQuery } from "react-query"; import { ISoftwareInstallResult, ISoftwareInstallResults, - SoftwareInstallStatus, + ISoftwareInstallStatus, } from "interfaces/software"; import softwareAPI from "services/entities/software"; @@ -18,13 +18,13 @@ import { IconNames } from "components/icons"; const baseClass = "software-install-details"; -const STATUS_ICONS: Record = { +const STATUS_ICONS: Record = { pending: "pending-outline", installed: "success-outline", failed: "error-outline", } as const; -const STATUS_PREDICATES: Record = { +const STATUS_PREDICATES: Record = { pending: "will install", installed: "installed", failed: "failed to install", diff --git a/frontend/pages/SoftwarePage/components/VersionCell/VersionCell.tsx b/frontend/pages/SoftwarePage/components/VersionCell/VersionCell.tsx index a8d563a700..a21e2dc7c7 100644 --- a/frontend/pages/SoftwarePage/components/VersionCell/VersionCell.tsx +++ b/frontend/pages/SoftwarePage/components/VersionCell/VersionCell.tsx @@ -1,24 +1,22 @@ import React from "react"; import { uniqueId } from "lodash"; -import { ISoftwareTitleVersion } from "interfaces/software"; - import TextCell from "components/TableContainer/DataTable/TextCell"; import ReactTooltip from "react-tooltip"; const baseClass = "version-cell"; -const generateText = (versions: ISoftwareTitleVersion[] | null) => { +const generateText = (versions: T[] | null) => { if (!versions) { - return ; + return ; } const text = versions.length !== 1 ? `${versions.length} versions` : versions[0].version; return ; }; -const generateTooltip = ( - versions: ISoftwareTitleVersion[], +const generateTooltip = ( + versions: T[], tooltipId: string ) => { if (!versions) { @@ -39,11 +37,13 @@ const generateTooltip = ( ); }; -interface IVersionCellProps { - versions: ISoftwareTitleVersion[] | null; +interface IVersionCellProps { + versions: T[] | null; } -const VersionCell = ({ versions }: IVersionCellProps) => { +const VersionCell = ({ + versions, +}: IVersionCellProps) => { const tooltipId = uniqueId(); // only one version, no need for tooltip diff --git a/frontend/pages/hosts/details/DeviceUserPage/DeviceUserPage.tsx b/frontend/pages/hosts/details/DeviceUserPage/DeviceUserPage.tsx index 629f4956b5..af31b28da3 100644 --- a/frontend/pages/hosts/details/DeviceUserPage/DeviceUserPage.tsx +++ b/frontend/pages/hosts/details/DeviceUserPage/DeviceUserPage.tsx @@ -334,7 +334,7 @@ const DeviceUserPage = ({ return (
- {isLoadingHost ? ( + {!host || isLoadingHost ? ( ) : (
@@ -406,15 +406,11 @@ const DeviceUserPage = ({ {isPremiumTier && ( diff --git a/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx b/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx index ebfa15770b..c540837c91 100644 --- a/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx +++ b/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx @@ -31,7 +31,7 @@ import { import { ILabel } from "interfaces/label"; import { IHostPolicy } from "interfaces/policy"; import { IQueryStats } from "interfaces/query_stats"; -import { ISoftware } from "interfaces/software"; +import Software, { IHostSoftware, ISoftware } from "interfaces/software"; import { DEFAULT_TARGETS_BY_TYPE } from "interfaces/target"; import { ITeam } from "interfaces/team"; import { @@ -92,6 +92,7 @@ import { getHostDeviceStatusUIState, } from "../helpers"; import WipeModal from "./modals/WipeModal"; +import SoftwareDetailsModal from "../cards/Software/SoftwareDetailsModal"; const baseClass = "host-details"; @@ -129,13 +130,11 @@ interface IHostDetailsSubNavItem { const DEFAULT_ACTIVITY_PAGE_SIZE = 8; const HostDetailsPage = ({ - route, router, location, params: { host_id }, }: IHostDetailsProps): JSX.Element => { const hostIdFromURL = parseInt(host_id, 10); - const routeTemplate = route?.path ?? ""; const queryParams = location.query; const { @@ -178,14 +177,16 @@ const HostDetailsPage = ({ const [showRefetchSpinner, setShowRefetchSpinner] = useState(false); const [schedule, setSchedule] = useState(); const [packsState, setPackState] = useState(); - const [hostSoftware, setHostSoftware] = useState([]); const [usersState, setUsersState] = useState<{ username: string }[]>([]); const [usersSearchString, setUsersSearchString] = useState(""); - const [pathname, setPathname] = useState(""); const [ hostMdmDeviceStatus, setHostMdmDeviceState, ] = useState("unlocked"); + const [ + selectedSoftwareDetails, + setSelectedSoftwareDetails, + ] = useState(null); // activity states const [activeActivityTab, setActiveActivityTab] = useState< @@ -334,7 +335,6 @@ const HostDetailsPage = ({ } return; // exit early because refectch is pending so we can avoid unecessary steps below } - setHostSoftware(returnedHost.software || []); setUsersState(returnedHost.users || []); setSchedule(schedule); if (returnedHost.pack_stats) { @@ -462,11 +462,6 @@ const HostDetailsPage = ({ } }, [location.pathname, host]); - // Used for back to software pathname - useEffect(() => { - setPathname(location.pathname + location.search); - }, [location]); - const summaryData = normalizeEmptyValues(pick(host, HOST_SUMMARY_DATA)); const aboutData = normalizeEmptyValues(pick(host, HOST_ABOUT_DATA)); @@ -855,15 +850,14 @@ const HostDetailsPage = ({ + setSelectedSoftwareDetails(software) + } /> {host?.platform === "darwin" && macadmins?.munki?.version && ( setShowWipeModal(false)} /> )} + {selectedSoftwareDetails && ( + setSelectedSoftwareDetails(null)} + /> + )} ); diff --git a/frontend/pages/hosts/details/cards/Activity/ActivityItems/InstalledSoftwareActivityItem/InstalledSoftwareActivityItem.tsx b/frontend/pages/hosts/details/cards/Activity/ActivityItems/InstalledSoftwareActivityItem/InstalledSoftwareActivityItem.tsx index fd0d7d3fa6..d141754d75 100644 --- a/frontend/pages/hosts/details/cards/Activity/ActivityItems/InstalledSoftwareActivityItem/InstalledSoftwareActivityItem.tsx +++ b/frontend/pages/hosts/details/cards/Activity/ActivityItems/InstalledSoftwareActivityItem/InstalledSoftwareActivityItem.tsx @@ -1,6 +1,6 @@ import React from "react"; -import { SoftwareInstallStatus } from "interfaces/software"; +import { ISoftwareInstallStatus } from "interfaces/software"; import { IHostActivityItemComponentPropsWithShowDetails } from "../../ActivityConfig"; import HostActivityItem from "../../HostActivityItem"; @@ -8,7 +8,7 @@ import ShowDetailsButton from "../../ShowDetailsButton"; const baseClass = "installed-software-activity-item"; -const STATUS_PREDICATES: Record = { +const STATUS_PREDICATES: Record = { failed: "failed to install", installed: "installed", pending: "told Fleet to install", @@ -21,7 +21,7 @@ export const getSoftwareInstallStatusPredicate = ( return STATUS_PREDICATES.pending; } return ( - STATUS_PREDICATES[status as SoftwareInstallStatus] || + STATUS_PREDICATES[status as ISoftwareInstallStatus] || STATUS_PREDICATES.pending ); }; diff --git a/frontend/pages/hosts/details/cards/Software/DeviceSoftwareTableConfig.tsx b/frontend/pages/hosts/details/cards/Software/DeviceSoftwareTableConfig.tsx new file mode 100644 index 0000000000..631f58bfcb --- /dev/null +++ b/frontend/pages/hosts/details/cards/Software/DeviceSoftwareTableConfig.tsx @@ -0,0 +1,90 @@ +import React from "react"; +import { CellProps, Column } from "react-table"; + +import { IHostSoftware, SOURCE_TYPE_CONVERSION } from "interfaces/software"; +import { IHeaderProps, IStringCellProps } from "interfaces/datatable_config"; + +import HeaderCell from "components/TableContainer/DataTable/HeaderCell/HeaderCell"; +import TextCell from "components/TableContainer/DataTable/TextCell"; + +import VulnerabilitiesCell from "pages/SoftwarePage/components/VulnerabilitiesCell"; +import VersionCell from "pages/SoftwarePage/components/VersionCell"; +import { getVulnerabilities } from "pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTitlesTableConfig"; +import SoftwareNameCell from "components/TableContainer/DataTable/SoftwareNameCell"; + +type ISoftwareTableConfig = Column; +type ITableHeaderProps = IHeaderProps; +type ITableStringCellProps = IStringCellProps; +type IInstalledVersionsCellProps = CellProps< + IHostSoftware, + IHostSoftware["installed_versions"] +>; +type IVulnerabilitiesCellProps = IInstalledVersionsCellProps; + +const formatSoftwareType = (source: string) => { + const DICT = SOURCE_TYPE_CONVERSION; + return DICT[source] || "Unknown"; +}; + +// interface ISoftwareTableHeadersProps {} + +export const generateSoftwareTableData = ( + software: IHostSoftware[] +): IHostSoftware[] => { + return software; +}; + +// NOTE: cellProps come from react-table +// more info here https://react-table.tanstack.com/docs/api/useTable#cell-properties +export const generateSoftwareTableHeaders = (): ISoftwareTableConfig[] => { + const tableHeaders: ISoftwareTableConfig[] = [ + { + Header: (cellProps: ITableHeaderProps) => ( + + ), + accessor: "name", + disableSortBy: false, + disableGlobalFilter: false, + Cell: (cellProps: ITableStringCellProps) => { + const { name, source } = cellProps.row.original; + return ; + }, + sortType: "caseInsensitive", + }, + { + Header: "Version", + disableSortBy: true, + // we use function as accessor because we have two columns that + // need to access the same data. This is not supported with a string + // accessor. + accessor: (originalRow) => originalRow.installed_versions, + Cell: (cellProps: IInstalledVersionsCellProps) => { + return ; + }, + }, + { + Header: (cellProps: ITableHeaderProps) => ( + + ), + disableSortBy: false, + disableGlobalFilter: true, + accessor: "source", + Cell: (cellProps: ITableStringCellProps) => ( + + ), + }, + { + Header: "Vulnerabilities", + accessor: (originalRow) => originalRow.installed_versions, + disableSortBy: true, + Cell: (cellProps: IVulnerabilitiesCellProps) => { + const vulnerabilities = getVulnerabilities(cellProps.cell.value ?? []); + return ; + }, + }, + ]; + + return tableHeaders; +}; + +export default { generateSoftwareTableHeaders, generateSoftwareTableData }; diff --git a/frontend/pages/hosts/details/cards/Software/HostSoftwareTable/HostSoftwareTable.tsx b/frontend/pages/hosts/details/cards/Software/HostSoftwareTable/HostSoftwareTable.tsx new file mode 100644 index 0000000000..017bbabbfc --- /dev/null +++ b/frontend/pages/hosts/details/cards/Software/HostSoftwareTable/HostSoftwareTable.tsx @@ -0,0 +1,124 @@ +import React, { useCallback } from "react"; +import { InjectedRouter } from "react-router"; + +import { IGetHostSoftwareResponse } from "services/entities/hosts"; +import { IGetDeviceSoftwareResponse } from "services/entities/device_user"; +import { getNextLocationPath } from "utilities/helpers"; + +import TableContainer from "components/TableContainer"; +import { ITableQueryData } from "components/TableContainer/TableContainer"; + +import EmptySoftwareTable from "pages/SoftwarePage/components/EmptySoftwareTable"; + +const DEFAULT_PAGE_SIZE = 20; + +const baseClass = "host-software-table"; + +interface IHostSoftwareTableProps { + tableConfig: any; // TODO: type + data: IGetHostSoftwareResponse | IGetDeviceSoftwareResponse; + isLoading: boolean; + router: InjectedRouter; + sortHeader: string; + sortDirection: "asc" | "desc"; + searchQuery: string; + page: number; + pagePath: string; +} + +const HostSoftwareTable = ({ + tableConfig, + data, + isLoading, + router, + sortHeader, + sortDirection, + searchQuery, + page, + pagePath, +}: IHostSoftwareTableProps) => { + const determineQueryParamChange = useCallback( + (newTableQuery: ITableQueryData) => { + const changedEntry = Object.entries(newTableQuery).find(([key, val]) => { + switch (key) { + case "searchQuery": + return val !== searchQuery; + case "sortDirection": + return val !== sortDirection; + case "sortHeader": + return val !== sortHeader; + case "pageIndex": + return val !== page; + default: + return false; + } + }); + return changedEntry?.[0] ?? ""; + }, + [page, searchQuery, sortDirection, sortHeader] + ); + + const generateNewQueryParams = useCallback( + (newTableQuery: ITableQueryData, changedParam: string) => { + const newQueryParam: Record = { + query: newTableQuery.searchQuery, + order_direction: newTableQuery.sortDirection, + order_key: newTableQuery.sortHeader, + page: changedParam === "pageIndex" ? newTableQuery.pageIndex : 0, + }; + + return newQueryParam; + }, + [] + ); + + // TODO: Look into useDebounceCallback with dependencies + const onQueryChange = useCallback( + async (newTableQuery: ITableQueryData) => { + // we want to determine which query param has changed in order to + // reset the page index to 0 if any other param has changed. + const changedParam = determineQueryParamChange(newTableQuery); + + // if nothing has changed, don't update the route. this can happen when + // this handler is called on the inital render. Can also happen when + // the filter dropdown is changed. That is handled on the onChange handler + // for the dropdown. + if (changedParam === "") return; + + const newRoute = getNextLocationPath({ + pathPrefix: pagePath, + routeTemplate: "", + queryParams: generateNewQueryParams(newTableQuery, changedParam), + }); + + router.replace(newRoute); + }, + [determineQueryParamChange, pagePath, generateNewQueryParams, router] + ); + + return ( +
+ ( + + )} + showMarkAllPages={false} + isAllPagesSelected={false} + searchable + /> +
+ ); +}; + +export default HostSoftwareTable; diff --git a/frontend/pages/hosts/details/cards/Software/HostSoftwareTable/index.ts b/frontend/pages/hosts/details/cards/Software/HostSoftwareTable/index.ts new file mode 100644 index 0000000000..62c0fb8cb5 --- /dev/null +++ b/frontend/pages/hosts/details/cards/Software/HostSoftwareTable/index.ts @@ -0,0 +1 @@ +export { default } from "./HostSoftwareTable"; diff --git a/frontend/pages/hosts/details/cards/Software/HostSoftwareTableConfig.tsx b/frontend/pages/hosts/details/cards/Software/HostSoftwareTableConfig.tsx new file mode 100644 index 0000000000..f9a4e354b2 --- /dev/null +++ b/frontend/pages/hosts/details/cards/Software/HostSoftwareTableConfig.tsx @@ -0,0 +1,174 @@ +import React from "react"; +import { InjectedRouter } from "react-router"; +import { CellProps, Column } from "react-table"; +import { cloneDeep } from "lodash"; + +import { + IHostSoftware, + SOURCE_TYPE_CONVERSION, + formatSoftwareType, +} from "interfaces/software"; +import { IHeaderProps, IStringCellProps } from "interfaces/datatable_config"; +import { IDropdownOption } from "interfaces/dropdownOption"; +import PATHS from "router/paths"; + +import HeaderCell from "components/TableContainer/DataTable/HeaderCell/HeaderCell"; +import TextCell from "components/TableContainer/DataTable/TextCell"; +import SoftwareNameCell from "components/TableContainer/DataTable/SoftwareNameCell"; +import DropdownCell from "components/TableContainer/DataTable/DropdownCell"; + +import VulnerabilitiesCell from "pages/SoftwarePage/components/VulnerabilitiesCell"; +import VersionCell from "pages/SoftwarePage/components/VersionCell"; +import { getVulnerabilities } from "pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTitlesTableConfig"; + +import InstallStatusCell from "./InstallStatusCell"; + +const DEFAULT_ACTION_OPTIONS: IDropdownOption[] = [ + { value: "showDetails", label: "Show details", disabled: false }, + { value: "install", label: "Install", disabled: false }, +]; + +type ISoftwareTableConfig = Column; +type ITableHeaderProps = IHeaderProps; +type ITableStringCellProps = IStringCellProps; +type IInstalledStatusCellProps = CellProps< + IHostSoftware, + IHostSoftware["status"] +>; +type IInstalledVersionsCellProps = CellProps< + IHostSoftware, + IHostSoftware["installed_versions"] +>; +type IVulnerabilitiesCellProps = IInstalledVersionsCellProps; +// type IActionsCellProps = CellProps; + +const generateActions = ( + packageToInstall: string | null, + softwareId: number, + installingSoftwareId: number | null +) => { + // this gives us a clean slate of the default actions so we can modify + // the options. + const actions = cloneDeep(DEFAULT_ACTION_OPTIONS); + + // TODO: when do we not show the options? + + // disable install option if software is already installing + if (softwareId === installingSoftwareId) { + const installAction = actions.find((action) => action.value === "install"); + if (installAction) { + installAction.disabled = true; + } + } + + return actions; +}; + +interface ISoftwareTableHeadersProps { + installingSoftwareId: number | null; + onSelectAction: (software: IHostSoftware, action: string) => void; + router: InjectedRouter; +} + +// NOTE: cellProps come from react-table +// more info here https://react-table.tanstack.com/docs/api/useTable#cell-properties +export const generateSoftwareTableHeaders = ({ + router, + installingSoftwareId, + onSelectAction, +}: ISoftwareTableHeadersProps): ISoftwareTableConfig[] => { + const tableHeaders: ISoftwareTableConfig[] = [ + { + Header: (cellProps: ITableHeaderProps) => ( + + ), + accessor: "name", + disableSortBy: false, + Cell: (cellProps: ITableStringCellProps) => { + const { id, name, source } = cellProps.row.original; + + const softwareTitleDetailsPath = PATHS.SOFTWARE_TITLE_DETAILS( + id.toString() + ); + + return ( + + ); + }, + }, + { + Header: "Install status", + disableSortBy: true, + accessor: "status", + Cell: (cellProps: IInstalledStatusCellProps) => { + const { original } = cellProps.row; + const { value } = cellProps.cell; + return value ? ( + + ) : null; + }, + }, + { + Header: "Version", + disableSortBy: true, + // we use function as accessor because we have two columns that + // need to access the same data. This is not supported with a string + // accessor. + accessor: (originalRow) => originalRow.installed_versions, + Cell: (cellProps: IInstalledVersionsCellProps) => { + return ; + }, + }, + { + Header: "Type", + disableSortBy: true, + accessor: "source", + Cell: (cellProps: ITableStringCellProps) => ( + formatSoftwareType({ source: cellProps.cell.value })} + /> + ), + }, + { + Header: "Vulnerabilities", + accessor: (originalRow) => originalRow.installed_versions, + disableSortBy: true, + Cell: (cellProps: IVulnerabilitiesCellProps) => { + const vulnerabilities = getVulnerabilities(cellProps.cell.value ?? []); + return ; + }, + }, + { + Header: "", + disableSortBy: true, + // the accessor here is insignificant, we just need it as its required + // but we don't use it. + accessor: "bundle_identifier", + Cell: (cellProps: ITableStringCellProps) => ( + onSelectAction(cellProps.row.original, action)} + /> + ), + }, + ]; + + return tableHeaders; +}; + +export default { generateSoftwareTableHeaders }; diff --git a/frontend/pages/hosts/details/cards/Software/InstallStatusCell/InstallStatusCell.tsx b/frontend/pages/hosts/details/cards/Software/InstallStatusCell/InstallStatusCell.tsx new file mode 100644 index 0000000000..0ca7d9214c --- /dev/null +++ b/frontend/pages/hosts/details/cards/Software/InstallStatusCell/InstallStatusCell.tsx @@ -0,0 +1,92 @@ +import React, { ReactNode } from "react"; + +import { ISoftwareInstallStatus } from "interfaces/software"; +import { dateAgo } from "utilities/date_format"; + +import Icon from "components/Icon"; +import TooltipWrapper from "components/TooltipWrapper"; + +const baseClass = "install-status-cell"; + +type IStatusValue = ISoftwareInstallStatus | "avaiableForInstall"; + +type IStatusDisplayConfig = { + iconName: "success" | "pending-outline" | "error" | "install"; + displayText: string; + tooltip: (softwareName: string | null, lastInstall?: string) => ReactNode; +}; + +const CELL_DISPLAY_OPTIONS: Record = { + installed: { + iconName: "success", + displayText: "Installed", + tooltip: (_, lastInstall) => ( + <> + Fleet installed software on these hosts. ( + {dateAgo(lastInstall as string)}) + + ), + }, + pending: { + iconName: "pending-outline", + displayText: "Pending", + tooltip: () => "Fleet will install software when the host comes online.", + }, + failed: { + iconName: "error", + displayText: "Failed", + tooltip: (_, lastInstall) => ( + <> + Fleet failed to install software ({dateAgo(lastInstall as string)} ago). + Select Actions > Software details to see more. + + ), + }, + avaiableForInstall: { + iconName: "install", + displayText: "Available for install", + tooltip: (softwareName) => ( + <> + {softwareName} can be installed on the host. Select{" "} + Actions > Install to install. + + ), + }, +}; + +interface IInstallStatusCellProps { + status: ISoftwareInstallStatus | null; + packageToInstall: string | null; + installedAt?: string; +} + +const InstallStatusCell = ({ + status, + packageToInstall, + installedAt, +}: IInstallStatusCellProps) => { + let displayStatus: IStatusValue; + if (status === null) { + displayStatus = "avaiableForInstall"; + } else { + displayStatus = status; + } + + const displayConfig = CELL_DISPLAY_OPTIONS[displayStatus]; + + return ( + +
+ + {displayConfig.displayText} +
+
+ ); +}; + +export default InstallStatusCell; diff --git a/frontend/pages/hosts/details/cards/Software/InstallStatusCell/_styles.scss b/frontend/pages/hosts/details/cards/Software/InstallStatusCell/_styles.scss new file mode 100644 index 0000000000..cadd24df28 --- /dev/null +++ b/frontend/pages/hosts/details/cards/Software/InstallStatusCell/_styles.scss @@ -0,0 +1,7 @@ +.install-status-cell { + &__status-content { + display: flex; + align-items: center; + gap: $pad-small; + } +} diff --git a/frontend/pages/hosts/details/cards/Software/InstallStatusCell/index.ts b/frontend/pages/hosts/details/cards/Software/InstallStatusCell/index.ts new file mode 100644 index 0000000000..fcd653e841 --- /dev/null +++ b/frontend/pages/hosts/details/cards/Software/InstallStatusCell/index.ts @@ -0,0 +1 @@ +export { default } from "./InstallStatusCell"; diff --git a/frontend/pages/hosts/details/cards/Software/Software.tsx b/frontend/pages/hosts/details/cards/Software/Software.tsx index b05c951d01..021a3eafd5 100644 --- a/frontend/pages/hosts/details/cards/Software/Software.tsx +++ b/frontend/pages/hosts/details/cards/Software/Software.tsx @@ -1,25 +1,22 @@ -import React, { useCallback, useContext, useMemo } from "react"; +import React, { useCallback, useContext, useMemo, useState } from "react"; import { InjectedRouter } from "react-router"; -import { Row } from "react-table"; -import PATHS from "router/paths"; -import { isEmpty } from "lodash"; +import { useQuery } from "react-query"; +import { AxiosError } from "axios"; -import { AppContext } from "context/app"; -import { ISoftware } from "interfaces/software"; -import { buildQueryStringFromParams } from "utilities/url"; +import hostAPI, { IGetHostSoftwareResponse } from "services/entities/hosts"; +import deviceAPI, { + IGetDeviceSoftwareResponse, +} from "services/entities/device_user"; +import { IHostSoftware, ISoftware } from "interfaces/software"; +import { DEFAULT_USE_QUERY_OPTIONS } from "utilities/constants"; +import { NotificationContext } from "context/notification"; -import TableContainer from "components/TableContainer"; -import { ITableQueryData } from "components/TableContainer/TableContainer"; import Card from "components/Card"; -import EmptySoftwareTable from "pages/SoftwarePage/components/EmptySoftwareTable"; -import { getNextLocationPath } from "utilities/helpers"; +import Spinner from "components/Spinner"; +import DataError from "components/DataError"; -import SoftwareVulnCount from "./SoftwareVulnCount"; - -import { - generateSoftwareTableHeaders, - generateSoftwareTableData, -} from "./SoftwareTableConfig"; +import { generateSoftwareTableHeaders } from "./HostSoftwareTableConfig"; +import HostSoftwareTable from "./HostSoftwareTable"; const baseClass = "software-card"; @@ -27,167 +24,144 @@ export interface ITableSoftware extends Omit { vulnerabilities: string[]; // for client-side search purposes, we only want an array of cve strings } -interface ISoftwareTableProps { - isLoading: boolean; - software: ISoftware[]; - deviceUser?: boolean; - deviceType?: string; - isSoftwareEnabled?: boolean; - router?: InjectedRouter; +interface ISoftwareCardProps { + /** This is the host id or the device token */ + id: number | string; + router: InjectedRouter; queryParams?: { - vulnerable?: string; page?: string; query?: string; order_key?: string; order_direction?: "asc" | "desc"; }; - routeTemplate?: string; pathname: string; - pathPrefix: string; -} - -interface IRowProps extends Row { - original: { - id?: number; - }; + onShowSoftwareDetails?: (software: IHostSoftware) => void; isSoftwareEnabled?: boolean; + isMyDevicePage?: boolean; } +const DEFAULT_SEARCH_QUERY = ""; const DEFAULT_SORT_DIRECTION = "desc"; const DEFAULT_SORT_HEADER = "name"; -const DEFAULT_PAGE_SIZE = 20; +const DEFAULT_PAGE = 0; -const SoftwareTable = ({ - isLoading, - software, - deviceUser, - deviceType, +const SoftwareCard = ({ + id, router, queryParams, - routeTemplate, - pathPrefix, pathname, -}: ISoftwareTableProps): JSX.Element => { - const { isSandboxMode, setFilteredSoftwarePath } = useContext(AppContext); + onShowSoftwareDetails, + isSoftwareEnabled = false, + isMyDevicePage = false, +}: ISoftwareCardProps) => { + const { renderFlash } = useContext(NotificationContext); - // Functions to avoid race conditions - const initialSearchQuery = (() => queryParams?.query ?? "")(); - const initialSortHeader = (() => queryParams?.order_key ?? "name")(); - const initialSortDirection = (() => - (queryParams?.order_direction as "asc" | "desc") ?? "asc")(); - const initialVulnFilter = (() => queryParams?.vulnerable === "true")(); - const initialPage = (() => - queryParams && queryParams.page ? parseInt(queryParams?.page, 10) : 0)(); + const [installingSoftwareId, setInstallingSoftwareId] = useState< + number | null + >(null); - // Never set as state as URL is source of truth - const searchQuery = initialSearchQuery; - const filterVuln = initialVulnFilter; - const page = initialPage; - const sortDirection = initialSortDirection; - const sortHeader = initialSortHeader; + const { + data: hostSoftwareRes, + isLoading: hostSoftwareLoading, + isError: hostSoftwareError, + isFetching: hostSoftwareFetching, + } = useQuery( + ["host-software", queryParams], + () => hostAPI.getHostSoftware(id as number), + { + ...DEFAULT_USE_QUERY_OPTIONS, + enabled: isSoftwareEnabled && !isMyDevicePage, + } + ); - // TODO: Look into useDebounceCallback with dependencies - const onQueryChange = useCallback( - async (newTableQuery: ITableQueryData) => { - const { - pageIndex: newPageIndex, - searchQuery: newSearchQuery, - sortDirection: newSortDirection, - sortHeader: newSortHeader, - } = newTableQuery; + const { + data: deviceSoftwareRes, + isLoading: deviceSoftwareLoading, + isError: deviceSoftwareError, + isFetching: deviceSoftwareFetching, + } = useQuery( + ["host-software", queryParams], + () => deviceAPI.getDeviceSoftware(id as string), + { + ...DEFAULT_USE_QUERY_OPTIONS, + enabled: isSoftwareEnabled && isMyDevicePage, + } + ); - // Rebuild queryParams to dispatch new browser location to react-router - const newQueryParams: { [key: string]: string | number | undefined } = {}; + const searchQuery = queryParams?.query ?? DEFAULT_SEARCH_QUERY; + const sortHeader = queryParams?.order_key ?? DEFAULT_SORT_HEADER; + const sortDirection = queryParams?.order_direction ?? DEFAULT_SORT_DIRECTION; + const page = queryParams?.page + ? parseInt(queryParams.page, 10) + : DEFAULT_PAGE; - if (!isEmpty(newSearchQuery)) { - newQueryParams.query = newSearchQuery; + const installHostSoftwarePackage = useCallback( + async (softwareId: number) => { + setInstallingSoftwareId(softwareId); + try { + await hostAPI.installHostSoftwarePackage(id as number, softwareId); + renderFlash( + "success", + "Software is installing or will install when the host comes online." + ); + } catch { + renderFlash("error", "Couldn't install. Please try again."); } + setInstallingSoftwareId(null); + }, + [id, renderFlash] + ); - newQueryParams.order_key = newSortHeader || DEFAULT_SORT_HEADER; - newQueryParams.order_direction = - newSortDirection || DEFAULT_SORT_DIRECTION; - newQueryParams.vulnerable = filterVuln ? "true" : "false"; // must set from URL - newQueryParams.page = newPageIndex; - // Reset page number to 0 for new filters - if ( - newSortDirection !== sortDirection || - newSortHeader !== sortHeader || - newSearchQuery !== searchQuery - ) { - newQueryParams.page = 0; + const onSelectAction = useCallback( + (software: IHostSoftware, action: string) => { + switch (action) { + case "install": + installHostSoftwarePackage(software.id); + break; + case "showDetails": + onShowSoftwareDetails?.(software); + break; + default: + break; } - - const locationPath = getNextLocationPath({ - pathPrefix, - routeTemplate, - queryParams: newQueryParams, - }); - - router?.replace(locationPath); }, - [sortHeader, sortDirection, searchQuery, filterVuln, router, routeTemplate] + [installHostSoftwarePackage, onShowSoftwareDetails] ); - const onClientSidePaginationChange = useCallback( - (pageIndex: number) => { - const locationPath = getNextLocationPath({ - pathPrefix, - routeTemplate, - queryParams: { - ...queryParams, - page: pageIndex, - vulnerable: filterVuln ? "true" : "false", - query: searchQuery, - order_direction: sortDirection, - order_key: sortHeader, - }, - }); - router?.replace(locationPath); - }, - [filterVuln, searchQuery, sortDirection, sortHeader] // Dependencies required for correct variable state - ); - - const tableSoftware = useMemo(() => generateSoftwareTableData(software), [ - software, - ]); - const tableHeaders = useMemo( - () => - generateSoftwareTableHeaders({ - deviceUser, - router, - setFilteredSoftwarePath, - pathname, - }), - [deviceUser, router, pathname] - ); - - const handleVulnFilterDropdownChange = (isFilterVulnerable: boolean) => { - const nextPath = getNextLocationPath({ - pathPrefix, - routeTemplate, - queryParams: { - ...queryParams, - page: 0, - vulnerable: isFilterVulnerable.toString(), - }, + const tableHeaders = useMemo(() => { + return generateSoftwareTableHeaders({ + router, + installingSoftwareId, + onSelectAction, }); - router?.replace(nextPath); - }; + }, [installingSoftwareId, router, onSelectAction]); - const handleRowSelect = (row: IRowProps) => { - if (deviceUser || !router) { - return; + const renderSoftwareTable = () => { + if (hostSoftwareLoading || deviceSoftwareLoading) { + return ; } - const hostsBySoftwareParams = { software_id: row.original.id }; + if (hostSoftwareError || deviceSoftwareError) { + return ; + } - const path = hostsBySoftwareParams - ? `${PATHS.MANAGE_HOSTS}?${buildQueryStringFromParams( - hostsBySoftwareParams - )}` - : PATHS.MANAGE_HOSTS; + if (!hostSoftwareRes || !deviceSoftwareRes) { + return null; + } - router.push(path); + return ( + + ); }; return ( @@ -198,52 +172,8 @@ const SoftwareTable = ({ className={baseClass} >

Software

- - {software?.length ? ( - <> - {software && ( - - )} - {software && ( -
- ( - - )} - showMarkAllPages={false} - isAllPagesSelected={false} - searchable - isClientSidePagination - onClientSidePaginationChange={onClientSidePaginationChange} - isClientSideFilter - disableMultiRowSelect={!deviceUser && !!router} // device user cannot view hosts by software - onSelectSingleRow={handleRowSelect} - /> -
- )} - - ) : ( - - )} + {renderSoftwareTable()} ); }; -export default SoftwareTable; +export default SoftwareCard; diff --git a/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/SoftwareDetailsModal.tsx b/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/SoftwareDetailsModal.tsx new file mode 100644 index 0000000000..372d5f3f7a --- /dev/null +++ b/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/SoftwareDetailsModal.tsx @@ -0,0 +1,126 @@ +import React from "react"; +import { Tab, TabList, TabPanel, Tabs } from "react-tabs"; + +import { + IHostSoftware, + ISoftwareInstallVersion, + formatSoftwareType, +} from "interfaces/software"; + +import Modal from "components/Modal"; +import TabsWrapper from "components/TabsWrapper"; +import Button from "components/buttons/Button"; +import DataSet from "components/DataSet"; +import { dateAgo } from "utilities/date_format"; + +const baseClass = "software-details-modal"; + +const generateVulnerabilitiesValue = (vulnerabilities: string[]) => { + const first3 = vulnerabilities.slice(0, 3); + const rest = vulnerabilities.slice(3); + + const first3Text = first3.join(", "); + const restText = `, +${rest.length} more`; + + return ( + <> + {`${first3Text}${rest.length > 0 ? restText : ""}`} + + ); +}; + +interface ISoftwareDetailsInfoProps { + installedVersion: ISoftwareInstallVersion; + source: string; + bundleIdentifier: string; +} + +const SoftwareDetailsInfo = ({ + installedVersion, + source, + bundleIdentifier, +}: ISoftwareDetailsInfoProps) => { + return ( +
+
+ + + + {installedVersion.last_opened_at && ( + + )} +
+
+ ( + <>{path} + ))} + /> +
+
+ +
+
+ ); +}; + +interface ISoftwareDetailsModalProps { + software: IHostSoftware; + onExit: () => void; +} + +const SoftwareDetailsModal = ({ + software, + onExit, +}: ISoftwareDetailsModalProps) => { + const renderSoftwareDetails = () => { + if ( + !software.installed_versions || + software.installed_versions.length === 0 + ) { + return null; + } + + return software.installed_versions.map((installedVersion) => { + return ( + + ); + }); + }; + + return ( + + <> + + + + Software details + Install Details + + {renderSoftwareDetails()} + test 2 + + +
+ +
+ +
+ ); +}; + +export default SoftwareDetailsModal; diff --git a/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/_styles.scss b/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/_styles.scss new file mode 100644 index 0000000000..3d7091408e --- /dev/null +++ b/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/_styles.scss @@ -0,0 +1,12 @@ +.software-details-modal { + &__details-info { + display: flex; + flex-direction: column; + gap: $pad-large; + } + + &__row { + display: flex; + gap: $pad-xxlarge; + } +} diff --git a/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/index.ts b/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/index.ts new file mode 100644 index 0000000000..8a8e498b21 --- /dev/null +++ b/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/index.ts @@ -0,0 +1 @@ +export { default } from "./SoftwareDetailsModal"; diff --git a/frontend/pages/hosts/details/cards/Software/SoftwareTableConfig.tsx b/frontend/pages/hosts/details/cards/Software/SoftwareTableConfig.tsx deleted file mode 100644 index b61680dc63..0000000000 --- a/frontend/pages/hosts/details/cards/Software/SoftwareTableConfig.tsx +++ /dev/null @@ -1,416 +0,0 @@ -import React from "react"; -import { InjectedRouter } from "react-router"; -import ReactTooltip from "react-tooltip"; - -import { formatDistanceToNow } from "date-fns"; - -import { ISoftware, SOURCE_TYPE_CONVERSION } from "interfaces/software"; -import PATHS from "router/paths"; - -import HeaderCell from "components/TableContainer/DataTable/HeaderCell/HeaderCell"; -import TextCell from "components/TableContainer/DataTable/TextCell"; -import LinkCell from "components/TableContainer/DataTable/LinkCell"; -import TooltipWrapper from "components/TooltipWrapper"; -import ViewAllHostsLink from "components/ViewAllHostsLink"; -import { DEFAULT_EMPTY_CELL_VALUE } from "utilities/constants"; -import { COLORS } from "styles/var/colors"; -import { getSoftwareBundleTooltipJSX } from "utilities/helpers"; - -interface IHeaderProps { - column: { - title: string; - isSortedDesc: boolean; - }; -} -interface ICellProps { - cell: { - value: number | string | string[]; - }; - row: { - original: ISoftware; - index: number; - }; -} - -interface IStringCellProps extends ICellProps { - cell: { - value: string; - }; -} - -interface IVulnCellProps extends ICellProps { - cell: { - value: string[]; - }; -} - -interface ILastUsedCellProps extends ICellProps { - cell: { - value: string; - }; -} - -interface IDataColumn { - title: string; - Header: ((props: IHeaderProps) => JSX.Element) | string; - accessor: string; - Cell: - | ((props: IStringCellProps) => JSX.Element) - | ((props: IVulnCellProps) => JSX.Element); - disableHidden?: boolean; - disableSortBy?: boolean; - disableGlobalFilter?: boolean; - sortType?: string; - // Filter can be used by react-table to render a filter input inside the column header - Filter?: () => null | JSX.Element; - filter?: string; // one of the enumerated `filterTypes` for react-table - // (see https://github.com/tannerlinsley/react-table/blob/master/src/filterTypes.js) - // or one of the custom `filterTypes` defined for the `useTable` instance (see `DataTable`) -} - -const formatSoftwareType = (source: string) => { - const DICT = SOURCE_TYPE_CONVERSION; - return DICT[source] || "Unknown"; -}; - -const condenseVulnerabilities = (vulns: string[]): string[] => { - const condensed = - (vulns?.length && vulns.length === 4 - ? vulns.slice(-4).reverse() - : vulns.slice(-3).reverse()) || []; - return vulns?.length > 4 - ? condensed.concat(`+${vulns?.length - 3} more`) - : condensed; -}; - -const renderBundleTooltip = (name: string, bundle: string) => ( - - - Bundle identifier: -
${bundle} -
- } - > - {name} - - -); - -interface IInstalledPathCellProps { - cell: { - value: string[]; - }; - row: { - original: ISoftware; - }; -} - -const condenseInstalledPaths = (installedPaths: string[]): string[] => { - if (!installedPaths?.length) { - return []; - } - const condensed = - installedPaths.length === 4 - ? installedPaths.slice(-4).reverse() - : installedPaths.slice(-3).reverse() || []; - return installedPaths.length > 4 - ? condensed.concat(`+${installedPaths.length - 3} more`) // TODO: confirm limit - : condensed; -}; - -const tooltipTextWithLineBreaks = (lines: string[]) => { - return lines.map((line) => { - return ( - - {line} -
-
- ); - }); -}; - -interface ISoftwareTableData extends Omit { - vulnerabilities: string[]; -} - -interface ISoftwareTableHeadersProps { - deviceUser?: boolean; - setFilteredSoftwarePath: (path: string) => void; - router?: InjectedRouter; - pathname: string; -} - -export const generateSoftwareTableData = ( - software: ISoftware[] -): ISoftwareTableData[] => { - return software.map((s) => { - return { - ...s, - vulnerabilities: s.vulnerabilities?.map((v) => v.cve) || [], - }; - }); -}; - -// NOTE: cellProps come from react-table -// more info here https://react-table.tanstack.com/docs/api/useTable#cell-properties -export const generateSoftwareTableHeaders = ({ - deviceUser = false, - setFilteredSoftwarePath, - router, - pathname, -}: ISoftwareTableHeadersProps): IDataColumn[] => { - const tableHeaders: IDataColumn[] = [ - { - title: "Name", - Header: (cellProps) => ( - - ), - accessor: "name", - disableSortBy: false, - disableGlobalFilter: false, - Cell: (cellProps: IStringCellProps) => { - const { id, name, bundle_identifier: bundle } = cellProps.row.original; - if (deviceUser) { - return bundle ? ( - renderBundleTooltip(name, bundle) - ) : ( - {name} - ); - } - - const onClickSoftware = (e: React.MouseEvent) => { - // Allows for button to be clickable in a clickable row - e.stopPropagation(); - setFilteredSoftwarePath(pathname); - router?.push(PATHS.SOFTWARE_VERSION_DETAILS(id.toString())); - }; - - return ( - - ); - }, - sortType: "caseInsensitive", - }, - { - title: "Version", - Header: "Version", - disableSortBy: true, - disableGlobalFilter: true, - accessor: "version", - Cell: (cellProps: IStringCellProps) => { - return ; - }, - }, - { - title: "Type", - Header: (cellProps) => ( - - ), - disableSortBy: false, - disableGlobalFilter: true, - accessor: "source", - Cell: (cellProps: IStringCellProps) => ( - - ), - }, - { - title: "Vulnerabilities", - Header: "Vulnerabilities", - accessor: "vulnerabilities", - disableSortBy: true, - disableGlobalFilter: false, - Filter: () => null, // input for this column filter outside of column header - filter: "hasLength", // filters out rows where vulnerabilities has no length if filter value is `true` - Cell: (cellProps: IVulnCellProps): JSX.Element => { - const vulnerabilities = cellProps.cell.value || []; - - const tooltipText = condenseVulnerabilities(vulnerabilities).map( - (value) => { - return ( - - {value} -
-
- ); - } - ); - - if (!vulnerabilities?.length) { - return ---; - } - return ( - <> - 1 ? "text-muted tooltip" : "" - }`} - data-tip - data-for={`vulnerabilities__${cellProps.row.original.id}`} - data-tip-disable={vulnerabilities.length <= 1} - > - {vulnerabilities.length === 1 - ? vulnerabilities[0] - : `${vulnerabilities.length} vulnerabilities`} - - - - {tooltipText} - - - - ); - }, - }, - { - title: "Last used", - Header: (cellProps) => ( - - ), - accessor: "last_opened_at", - Cell: (cellProps: ILastUsedCellProps): JSX.Element => { - const lastUsed = cellProps.cell.value - ? `${formatDistanceToNow(Date.parse(cellProps.cell.value))} ago` - : "Unavailable"; - const hasLastUsed = lastUsed !== "Unavailable"; - return ( - <> - - {lastUsed} - - - - Last used information
- is only available for the
- Application (macOS)
- software type. -
-
- - ); - }, - sortType: "dateStrings", - }, - { - title: "File path", - Header: () => { - return ( - - This is where the software is
- located on this host. - - } - > - File path -
- ); - }, - disableSortBy: true, - accessor: "installed_paths", - Cell: (cellProps: IInstalledPathCellProps): JSX.Element => { - const numInstalledPaths = cellProps.cell.value?.length || 0; - const installedPaths = condenseInstalledPaths( - cellProps.cell.value || [] - ); - if (installedPaths.length) { - const tooltipText = tooltipTextWithLineBreaks(installedPaths); - return ( - <> - 1 ? "text-muted tooltip" : "" - }`} - data-tip - data-for={`installed_paths__${cellProps.row.original.id}`} - data-tip-disable={installedPaths.length <= 1} - > - {numInstalledPaths === 1 - ? installedPaths[0] - : `${numInstalledPaths} paths`} - - - {tooltipText} - - - ); - } - return {DEFAULT_EMPTY_CELL_VALUE}; - }, - }, - { - title: "", - Header: "", - disableSortBy: true, - disableGlobalFilter: true, - accessor: "linkToFilteredHosts", - Cell: (cellProps: IStringCellProps) => { - return ( - - ); - }, - disableHidden: true, - }, - ]; - - // Device user cannot view all hosts software - if (deviceUser) { - tableHeaders.pop(); - } - - return tableHeaders; -}; - -export default { generateSoftwareTableHeaders, generateSoftwareTableData }; diff --git a/frontend/pages/hosts/details/cards/Software/SoftwareVulnCount/SoftwareVulnCount.tsx b/frontend/pages/hosts/details/cards/Software/SoftwareVulnCount/SoftwareVulnCount.tsx deleted file mode 100644 index 0fdace872b..0000000000 --- a/frontend/pages/hosts/details/cards/Software/SoftwareVulnCount/SoftwareVulnCount.tsx +++ /dev/null @@ -1,41 +0,0 @@ -import React from "react"; - -import { ISoftware } from "interfaces/software"; -import Icon from "components/Icon/Icon"; -import InfoBanner from "components/InfoBanner"; - -const baseClass = "software-vuln-count"; - -interface ISoftwareVulnCountProps { - softwareList: ISoftware[]; - deviceUser?: boolean; -} - -const SoftwareVulnCount = ({ - softwareList, - deviceUser, -}: ISoftwareVulnCountProps): JSX.Element => { - const vulnCount = softwareList.reduce((sum, software) => { - return software.vulnerabilities?.length ? sum + 1 : sum; - }, 0); - return vulnCount ? ( - -
- - {vulnCount === 1 - ? "1 software item with vulnerabilities detected" - : `${vulnCount} software items with vulnerabilities detected`} -
- {!deviceUser && ( -

- Click a vulnerable item below to see the associated Common - Vulnerabilites and Exposures (CVEs). -

- )} -
- ) : ( - <> - ); -}; - -export default SoftwareVulnCount; diff --git a/frontend/pages/hosts/details/cards/Software/SoftwareVulnCount/_styles.scss b/frontend/pages/hosts/details/cards/Software/SoftwareVulnCount/_styles.scss deleted file mode 100644 index 654886d39f..0000000000 --- a/frontend/pages/hosts/details/cards/Software/SoftwareVulnCount/_styles.scss +++ /dev/null @@ -1,11 +0,0 @@ -.software-vuln-count { - &__count { - display: flex; - font-weight: $bold; - gap: $pad-small; - } - - p { - margin-left: $pad-large; // Align second line with first line and not with icon - } -} diff --git a/frontend/pages/hosts/details/cards/Software/SoftwareVulnCount/index.ts b/frontend/pages/hosts/details/cards/Software/SoftwareVulnCount/index.ts deleted file mode 100644 index fca34bbc90..0000000000 --- a/frontend/pages/hosts/details/cards/Software/SoftwareVulnCount/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from "./SoftwareVulnCount"; diff --git a/frontend/services/entities/device_user.ts b/frontend/services/entities/device_user.ts index 4175a97709..7d5152e527 100644 --- a/frontend/services/entities/device_user.ts +++ b/frontend/services/entities/device_user.ts @@ -1,10 +1,23 @@ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ +import { + createMockGetHostSoftwareResponse, + createMockHostSoftware, +} from "__mocks__/hostMock"; import { IDeviceUserResponse } from "interfaces/host"; +import { IHostSoftware } from "interfaces/software"; import sendRequest from "services"; import endpoints from "utilities/endpoints"; export type ILoadHostDetailsExtension = "device_mapping" | "macadmins"; +export interface IGetDeviceSoftwareResponse { + software: IHostSoftware[]; + meta: { + has_next_results: boolean; + has_previous_results: boolean; + }; +} + export default { loadHostDetails: (deviceAuthToken: string): Promise => { const { DEVICE_USER_DETAILS } = endpoints; @@ -27,4 +40,21 @@ export default { return sendRequest("POST", path); }, + + getDeviceSoftware: ( + deviceAuthToken: string + ): Promise => { + const { DEVICE_SOFTWARE } = endpoints; + + // TODO: remove when API ready + // return sendRequest("GET", DEVICE_SOFTWARE(deviceAuthToken)); + + return new Promise((resolve) => { + resolve( + createMockGetHostSoftwareResponse({ + software: [createMockHostSoftware()], + }) + ); + }); + }, }; diff --git a/frontend/services/entities/hosts.ts b/frontend/services/entities/hosts.ts index 557e4fe18e..f65be46d33 100644 --- a/frontend/services/entities/hosts.ts +++ b/frontend/services/entities/hosts.ts @@ -9,7 +9,7 @@ import { reconcileMutuallyInclusiveHostParams, } from "utilities/url"; import { SelectedPlatform } from "interfaces/platform"; -import { ISoftwareTitle, ISoftware } from "interfaces/software"; +import { ISoftwareTitle, ISoftware, IHostSoftware } from "interfaces/software"; import { DiskEncryptionStatus, BootstrapPackageStatus, @@ -19,6 +19,10 @@ import { } from "interfaces/mdm"; import { IMunkiIssuesAggregate } from "interfaces/macadmins"; import { PolicyResponse } from "utilities/constants"; +import { + createMockGetHostSoftwareResponse, + createMockHostSoftware, +} from "__mocks__/hostMock"; export interface ISortOption { key: string; @@ -140,6 +144,14 @@ export interface IActionByFilter { vulnerability?: string; } +export interface IGetHostSoftwareResponse { + software: IHostSoftware[]; + meta: { + has_next_results: boolean; + has_previous_results: boolean; + }; +} + export type ILoadHostDetailsExtension = "device_mapping" | "macadmins"; const LABEL_PREFIX = "labels/"; @@ -530,4 +542,27 @@ export default { return sendRequest("POST", HOST_RESEND_PROFILE(hostId, profileUUID)); }, + + getHostSoftware: (hostId: number): Promise => { + const { HOST_SOFTWARE } = endpoints; + + // TODO: remove when API ready + // return sendRequest("GET", HOST_SOFTWARE(hostId)); + + return new Promise((resolve) => { + resolve( + createMockGetHostSoftwareResponse({ + software: [createMockHostSoftware()], + }) + ); + }); + }, + + installHostSoftwarePackage: (hostId: number, softwareId: number) => { + const { HOST_SOFTWARE_PACKAGE_INSTALL } = endpoints; + return sendRequest( + "POST", + HOST_SOFTWARE_PACKAGE_INSTALL(hostId, softwareId) + ); + }, }; diff --git a/frontend/services/entities/software.ts b/frontend/services/entities/software.ts index f27bb56502..a55e08e399 100644 --- a/frontend/services/entities/software.ts +++ b/frontend/services/entities/software.ts @@ -5,7 +5,6 @@ import endpoints from "utilities/endpoints"; import { ISoftwareResponse, ISoftwareCountResponse, - IGetSoftwareByIdResponse, ISoftwareVersion, ISoftwareTitle, } from "interfaces/software"; @@ -14,7 +13,6 @@ import { IAddSoftwareFormData } from "pages/SoftwarePage/components/AddSoftwareF import { createMockSoftwarePackage, createMockSoftwareTitle, - createMockSoftwareTitleResponse, } from "__mocks__/softwareMock"; export interface ISoftwareApiParams { diff --git a/frontend/utilities/date_format/index.ts b/frontend/utilities/date_format/index.ts index 356eaef6f2..53de0b5498 100644 --- a/frontend/utilities/date_format/index.ts +++ b/frontend/utilities/date_format/index.ts @@ -4,3 +4,7 @@ import { formatDistanceToNow } from "date-fns"; export const uploadedFromNow = (date: string) => { return `Uploaded ${formatDistanceToNow(new Date(date))} ago`; }; + +export const dateAgo = (date: string) => { + return `${formatDistanceToNow(new Date(date))} ago`; +}; diff --git a/frontend/utilities/endpoints.ts b/frontend/utilities/endpoints.ts index 1ecc1c726a..b0d4d9c7da 100644 --- a/frontend/utilities/endpoints.ts +++ b/frontend/utilities/endpoints.ts @@ -1,3 +1,5 @@ +import software from "interfaces/software"; + const API_VERSION = "latest"; export default { @@ -15,13 +17,7 @@ export default { CONFIRM_EMAIL_CHANGE: (token: string): string => { return `/${API_VERSION}/fleet/email/change/${token}`; }, - DEVICE_USER_DETAILS: `/${API_VERSION}/fleet/device`, - DEVICE_USER_MDM_ENROLLMENT_PROFILE: (token: string): string => { - return `/${API_VERSION}/fleet/device/${token}/mdm/apple/manual_enrollment_profile`; - }, - DEVICE_USER_RESET_ENCRYPTION_KEY: (token: string): string => { - return `/${API_VERSION}/fleet/device/${token}/rotate_encryption_key`; - }, + DOWNLOAD_INSTALLER: `/${API_VERSION}/fleet/download_installer`, ENABLE_USER: (id: number): string => { return `/${API_VERSION}/fleet/users/${id}/enable`; @@ -31,6 +27,17 @@ export default { GLOBAL_POLICIES: `/${API_VERSION}/fleet/policies`, GLOBAL_SCHEDULE: `/${API_VERSION}/fleet/schedule`, + // Device endpoints + DEVICE_USER_DETAILS: `/${API_VERSION}/fleet/device`, + DEVICE_SOFTWARE: (token: string) => + `/${API_VERSION}/fleet/devices/${token}/software`, + DEVICE_USER_RESET_ENCRYPTION_KEY: (token: string): string => { + return `/${API_VERSION}/fleet/device/${token}/rotate_encryption_key`; + }, + DEVICE_USER_MDM_ENROLLMENT_PROFILE: (token: string): string => { + return `/${API_VERSION}/fleet/device/${token}/mdm/apple/manual_enrollment_profile`; + }, + // Host endpoints HOST_SUMMARY: `/${API_VERSION}/fleet/host_summary`, HOST_QUERY_REPORT: (hostId: number, queryId: number) => @@ -46,6 +53,9 @@ export default { HOST_WIPE: (id: number) => `/${API_VERSION}/fleet/hosts/${id}/wipe`, HOST_RESEND_PROFILE: (hostId: number, profileUUID: string) => `/${API_VERSION}/fleet/hosts/${hostId}/configuration_profiles/resend/${profileUUID}`, + HOST_SOFTWARE: (id: number) => `/${API_VERSION}/fleet/hosts/${id}/software`, + HOST_SOFTWARE_PACKAGE_INSTALL: (hostId: number, softwareId: number) => + `/${API_VERSION}/fleet/hosts/${hostId}/software/install/${softwareId}`, INVITES: `/${API_VERSION}/fleet/invites`, @@ -133,6 +143,8 @@ export default { `/${API_VERSION}/fleet/software/packages/${id}`, SOFTWARE_INSTALL_RESULTS: (uuid: string) => `/${API_VERSION}/fleet/software/install/results/${uuid}`, + SOFTWARE_PACKAGE_INSTALL: (id: number) => + `/${API_VERSION}/fleet/software/packages/${id}`, // AI endpoints AUTOFILL_POLICY: `/${API_VERSION}/fleet/autofill/policy`, From 7f803e46295cb41e11f6b3322c227d3629d35256 Mon Sep 17 00:00:00 2001 From: Gabriel Hernandez Date: Fri, 10 May 2024 16:18:24 +0100 Subject: [PATCH 30/56] UI software install feature integrations and polish (#18906) relates to #18677 implement UI API integrations and polish tasks for the software install feature. These include various tasks to finish up this feature on the UI and ensure its working correctly - [x] Manual QA for all new/changed functionality --- frontend/components/DataSet/DataSet.tsx | 8 +- frontend/components/Editor/Editor.tsx | 15 + frontend/components/Editor/_styles.scss | 1 + .../SoftwareNameCell/SoftwareNameCell.tsx | 7 + .../DataTable/SoftwareNameCell/_styles.scss | 6 + frontend/interfaces/software.ts | 8 +- frontend/pages/SoftwarePage/SoftwarePage.tsx | 1 + .../AdvancedOptionsModal.tsx | 10 +- .../SoftwareTitleDetailsPage.tsx | 12 +- .../SoftwareTitlesTableConfig.tsx | 9 +- .../AddSoftwareForm/AddSoftwareForm.tsx | 25 +- .../components/AddSoftwareForm/helpers.ts | 5 - .../AddSoftwareModal/AddSoftwareModal.tsx | 29 +- .../HostDetailsPage/HostDetailsPage.tsx | 2 - .../Software/HostSoftwareTableConfig.tsx | 40 ++- .../InstallStatusCell/InstallStatusCell.tsx | 13 +- .../Software/InstallStatusCell/_styles.scss | 4 + .../hosts/details/cards/Software/Software.tsx | 135 ++++++--- .../SoftwareDetailsModal.tsx | 94 +++--- .../SoftwareDetailsModal/_styles.scss | 18 ++ .../hosts/details/cards/Software/_styles.scss | 280 +----------------- frontend/services/entities/device_user.ts | 33 +-- frontend/services/entities/hosts.ts | 29 +- frontend/services/entities/software.ts | 16 +- 24 files changed, 348 insertions(+), 452 deletions(-) diff --git a/frontend/components/DataSet/DataSet.tsx b/frontend/components/DataSet/DataSet.tsx index 25a4b78bff..9e1b138178 100644 --- a/frontend/components/DataSet/DataSet.tsx +++ b/frontend/components/DataSet/DataSet.tsx @@ -1,15 +1,19 @@ import React from "react"; +import classnames from "classnames"; const baseClass = "data-set"; interface IDataSetProps { title: React.ReactNode; value: React.ReactNode; + className?: string; } -const DataSet = ({ title, value }: IDataSetProps) => { +const DataSet = ({ title, value, className }: IDataSetProps) => { + const classNames = classnames(baseClass, className); + return ( -
+
{title}
{value}
diff --git a/frontend/components/Editor/Editor.tsx b/frontend/components/Editor/Editor.tsx index 6b2fd919c6..d5c39d3d01 100644 --- a/frontend/components/Editor/Editor.tsx +++ b/frontend/components/Editor/Editor.tsx @@ -1,4 +1,5 @@ import classnames from "classnames"; +import TooltipWrapper from "components/TooltipWrapper"; import React, { ReactNode } from "react"; import AceEditor from "react-ace"; @@ -7,6 +8,7 @@ const baseClass = "editor"; interface IEditorProps { focus?: boolean; label?: string; + labelTooltip?: string; error?: string | null; readOnly?: boolean; /** @@ -42,6 +44,7 @@ interface IEditorProps { const Editor = ({ helpText, label, + labelTooltip, error, focus, value, @@ -67,6 +70,18 @@ const Editor = ({ return null; } + if (labelTooltip) { + return ( + + {labelText} + + ); + } + return
{labelText}
; }; diff --git a/frontend/components/Editor/_styles.scss b/frontend/components/Editor/_styles.scss index 22fbacfd33..676172697d 100644 --- a/frontend/components/Editor/_styles.scss +++ b/frontend/components/Editor/_styles.scss @@ -3,6 +3,7 @@ &__label { font-size: $x-small; font-weight: $bold; + margin-bottom: $pad-small; &--error { color: $core-vibrant-red; diff --git a/frontend/components/TableContainer/DataTable/SoftwareNameCell/SoftwareNameCell.tsx b/frontend/components/TableContainer/DataTable/SoftwareNameCell/SoftwareNameCell.tsx index 79e25e8890..7f87581b55 100644 --- a/frontend/components/TableContainer/DataTable/SoftwareNameCell/SoftwareNameCell.tsx +++ b/frontend/components/TableContainer/DataTable/SoftwareNameCell/SoftwareNameCell.tsx @@ -1,6 +1,8 @@ import React from "react"; import { InjectedRouter } from "react-router"; +import Icon from "components/Icon"; + import SoftwareIcon from "pages/SoftwarePage/components/icons/SoftwareIcon"; import LinkCell from "../LinkCell"; @@ -12,6 +14,7 @@ interface ISoftwareNameCellProps { source: string; path?: string; router?: InjectedRouter; + hasPackage?: boolean; } const SoftwareNameCell = ({ @@ -19,6 +22,7 @@ const SoftwareNameCell = ({ source, path, router, + hasPackage = false, }: ISoftwareNameCellProps) => { // NO path or router means it's not clickable. return // a non-clickable cell early @@ -46,6 +50,9 @@ const SoftwareNameCell = ({ <> {name} + {hasPackage && ( + + )} } /> diff --git a/frontend/components/TableContainer/DataTable/SoftwareNameCell/_styles.scss b/frontend/components/TableContainer/DataTable/SoftwareNameCell/_styles.scss index a94f994132..d387075dd1 100644 --- a/frontend/components/TableContainer/DataTable/SoftwareNameCell/_styles.scss +++ b/frontend/components/TableContainer/DataTable/SoftwareNameCell/_styles.scss @@ -13,4 +13,10 @@ border: 1px solid $ui-fleet-black-10; border-radius: 8px; } + + &__install-icon { + // TODO: we do not want to use !important but have to for now. This is + // the same issue as the .software-name-cell class display value. + display: inline-flex !important; + } } diff --git a/frontend/interfaces/software.ts b/frontend/interfaces/software.ts index f6a75eac85..64737c89ba 100644 --- a/frontend/interfaces/software.ts +++ b/frontend/interfaces/software.ts @@ -67,7 +67,7 @@ export interface ISoftwarePackage { export interface ISoftwareTitle { id: number; name: string; - software_package: ISoftwarePackage | string | null; + software_package: ISoftwarePackage | null; versions_count: number; source: string; hosts_count: number; @@ -190,16 +190,16 @@ export interface ISoftwareLastInstall { export interface ISoftwareInstallVersion { version: string; last_opened_at: string | null; - vulnerabilities: string[]; + vulnerabilities: string[] | null; installed_paths: string[]; } export interface IHostSoftware { id: number; name: string; - package_available_for_install: string | null; + package_available_for_install?: string | null; source: string; - bundle_identifier: string; + bundle_identifier?: string; status: ISoftwareInstallStatus | null; last_install: ISoftwareLastInstall | null; installed_versions: ISoftwareInstallVersion[] | null; diff --git a/frontend/pages/SoftwarePage/SoftwarePage.tsx b/frontend/pages/SoftwarePage/SoftwarePage.tsx index 4a16bb9598..0916ba8c97 100644 --- a/frontend/pages/SoftwarePage/SoftwarePage.tsx +++ b/frontend/pages/SoftwarePage/SoftwarePage.tsx @@ -422,6 +422,7 @@ const SoftwarePage = ({ children, router, location }: ISoftwarePageProps) => { {showAddSoftwareModal && ( )} diff --git a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/AdvancedOptionsModal/AdvancedOptionsModal.tsx b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/AdvancedOptionsModal/AdvancedOptionsModal.tsx index 30395a89d1..008e78f514 100644 --- a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/AdvancedOptionsModal/AdvancedOptionsModal.tsx +++ b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/AdvancedOptionsModal/AdvancedOptionsModal.tsx @@ -31,11 +31,15 @@ const AdvancedOptionsModal = ({ add again.

- {preInstallQuery && (
diff --git a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwareTitleDetailsPage.tsx b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwareTitleDetailsPage.tsx index bc6f91a539..792c17bdcb 100644 --- a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwareTitleDetailsPage.tsx +++ b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwareTitleDetailsPage.tsx @@ -7,7 +7,6 @@ import { RouteComponentProps } from "react-router"; import { AxiosError } from "axios"; import useTeamIdParam from "hooks/useTeamIdParam"; -import { createMockSoftwarePackage } from "__mocks__/softwareMock"; import { AppContext } from "context/app"; @@ -102,9 +101,14 @@ const SoftwareTitleDetailsPage = ({ [handleTeamChange] ); + const hasPermission = Boolean( + isOnGlobalTeam || isTeamAdmin || isTeamMaintainer || isTeamObserver + ); + const hasSoftwarePackage = softwareTitle && softwareTitle.software_package; const showPackageCard = currentTeamId !== APP_CONTEXT_ALL_TEAMS_ID && - (isOnGlobalTeam || isTeamAdmin || isTeamMaintainer || isTeamObserver); + hasPermission && + hasSoftwarePackage; const renderContent = () => { if (isSoftwareTitleLoading) { @@ -145,9 +149,9 @@ const SoftwareTitleDetailsPage = ({ name={softwareTitle.name} source={softwareTitle.source} /> - {showPackageCard && ( + {showPackageCard && softwareTitle.software_package && ( diff --git a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTitlesTableConfig.tsx b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTitlesTableConfig.tsx index e4626c4d95..d45403fc60 100644 --- a/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTitlesTableConfig.tsx +++ b/frontend/pages/SoftwarePage/SoftwareTitles/SoftwareTable/SoftwareTitlesTableConfig.tsx @@ -13,8 +13,6 @@ import TextCell from "components/TableContainer/DataTable/TextCell"; import ViewAllHostsLink from "components/ViewAllHostsLink"; import SoftwareNameCell from "components/TableContainer/DataTable/SoftwareNameCell"; -import IconCell from "pages/SoftwarePage/components/IconCell"; - import VersionCell from "../../components/VersionCell"; import VulnerabilitiesCell from "../../components/VulnerabilitiesCell"; @@ -23,10 +21,6 @@ import VulnerabilitiesCell from "../../components/VulnerabilitiesCell"; type ISoftwareTitlesTableConfig = Column; type ITableStringCellProps = IStringCellProps; -type ISoftwarePackageCellProps = CellProps< - ISoftwareTitle, - ISoftwareTitle["software_package"] ->; type IVersionsCellProps = CellProps; type IVulnerabilitiesCellProps = IVersionsCellProps; type IHostCountCellProps = CellProps< @@ -69,7 +63,7 @@ const generateTableHeaders = ( disableSortBy: false, accessor: "name", Cell: (cellProps: ITableStringCellProps) => { - const { id, name, source } = cellProps.row.original; + const { id, name, source, software_package } = cellProps.row.original; const teamQueryParam = buildQueryStringFromParams({ team_id: teamId }); const softwareTitleDetailsPath = `${PATHS.SOFTWARE_TITLE_DETAILS( @@ -82,6 +76,7 @@ const generateTableHeaders = ( source={source} path={softwareTitleDetailsPath} router={router} + hasPackage={Boolean(software_package)} /> ); }, diff --git a/frontend/pages/SoftwarePage/components/AddSoftwareForm/AddSoftwareForm.tsx b/frontend/pages/SoftwarePage/components/AddSoftwareForm/AddSoftwareForm.tsx index 5c619e9687..05ece55866 100644 --- a/frontend/pages/SoftwarePage/components/AddSoftwareForm/AddSoftwareForm.tsx +++ b/frontend/pages/SoftwarePage/components/AddSoftwareForm/AddSoftwareForm.tsx @@ -1,19 +1,16 @@ import React, { useState } from "react"; -// @ts-ignore -import InputField from "components/forms/fields/InputField"; +import getInstallScript from "utilities/software_install_scripts"; + import Spinner from "components/Spinner"; import Button from "components/buttons/Button"; import FileUploader from "components/FileUploader"; import Graphic from "components/Graphic"; +import Editor from "components/Editor"; import AddSoftwareAdvancedOptions from "../AddSoftwareAdvancedOptions"; -import { - generateFormValidation, - getFileDetails, - getInstallScript, -} from "./helpers"; +import { generateFormValidation, getFileDetails } from "./helpers"; const baseClass = "add-software-form"; @@ -93,7 +90,7 @@ const AddSoftwareForm = ({ const newData = { ...formData, software: file, - installScript: getInstallScript(file), + installScript: getInstallScript(file.name), }; setFormData(newData); setFormValidation( @@ -180,13 +177,15 @@ const AddSoftwareForm = ({ } /> {formData.software && ( - )} { platform: getPlatformDisplayName(file), }; }; - -export const getInstallScript = (file: File) => { - // TODO: get this dynamically - return `sudo installer -pkg ${file.name} -target /`; -}; diff --git a/frontend/pages/SoftwarePage/components/AddSoftwareModal/AddSoftwareModal.tsx b/frontend/pages/SoftwarePage/components/AddSoftwareModal/AddSoftwareModal.tsx index e2233964d6..cec958b042 100644 --- a/frontend/pages/SoftwarePage/components/AddSoftwareModal/AddSoftwareModal.tsx +++ b/frontend/pages/SoftwarePage/components/AddSoftwareModal/AddSoftwareModal.tsx @@ -1,9 +1,12 @@ import React, { useContext, useEffect, useState } from "react"; +import { InjectedRouter } from "react-router"; +import PATHS from "router/paths"; import { APP_CONTEXT_ALL_TEAMS_ID } from "interfaces/team"; import { getErrorReason } from "interfaces/errors"; import softwareAPI from "services/entities/software"; import { NotificationContext } from "context/notification"; +import { buildQueryStringFromParams } from "utilities/url"; import Modal from "components/Modal"; import Button from "components/buttons/Button"; @@ -38,10 +41,15 @@ const AllTeamsMessage = ({ onExit }: IAllTeamsMessageProps) => { interface IAddSoftwareModalProps { teamId: number; + router: InjectedRouter; onExit: () => void; } -const AddSoftwareModal = ({ teamId, onExit }: IAddSoftwareModalProps) => { +const AddSoftwareModal = ({ + teamId, + router, + onExit, +}: IAddSoftwareModalProps) => { const { renderFlash } = useContext(NotificationContext); const [isUploading, setIsUploading] = useState(false); @@ -50,7 +58,8 @@ const AddSoftwareModal = ({ teamId, onExit }: IAddSoftwareModalProps) => { const beforeUnloadHandler = (e: BeforeUnloadEvent) => { e.preventDefault(); - // Included for legacy support, e.g. Chrome/Edge < 119 + // Next line with e.returnValue is included for legacy support + // e.g.Chrome / Edge < 119 e.returnValue = true; }; @@ -76,9 +85,23 @@ const AddSoftwareModal = ({ teamId, onExit }: IAddSoftwareModalProps) => { try { await softwareAPI.addSoftwarePackage(formData, teamId); - renderFlash("success", "Software added successfully!"); // TODO: change message + renderFlash( + "success", + <> + {formData.software?.name} successfully added. Go to Host details page + to install software. + + ); + onExit(); + router.push( + `${PATHS.SOFTWARE_TITLES}?${buildQueryStringFromParams({ + available_for_install: true, + team_id: teamId, + })}` + ); } catch (e) { renderFlash("error", getErrorReason(e)); + onExit(); } setIsUploading(false); diff --git a/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx b/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx index c540837c91..001b59ffca 100644 --- a/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx +++ b/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx @@ -97,12 +97,10 @@ import SoftwareDetailsModal from "../cards/Software/SoftwareDetailsModal"; const baseClass = "host-details"; interface IHostDetailsProps { - route: RouteProps; router: InjectedRouter; // v3 location: { pathname: string; query: { - vulnerable?: string; page?: string; query?: string; order_key?: string; diff --git a/frontend/pages/hosts/details/cards/Software/HostSoftwareTableConfig.tsx b/frontend/pages/hosts/details/cards/Software/HostSoftwareTableConfig.tsx index f9a4e354b2..307b91ec2e 100644 --- a/frontend/pages/hosts/details/cards/Software/HostSoftwareTableConfig.tsx +++ b/frontend/pages/hosts/details/cards/Software/HostSoftwareTableConfig.tsx @@ -5,10 +5,14 @@ import { cloneDeep } from "lodash"; import { IHostSoftware, - SOURCE_TYPE_CONVERSION, + ISoftwareInstallStatus, formatSoftwareType, } from "interfaces/software"; -import { IHeaderProps, IStringCellProps } from "interfaces/datatable_config"; +import { + IHeaderProps, + INumberCellProps, + IStringCellProps, +} from "interfaces/datatable_config"; import { IDropdownOption } from "interfaces/dropdownOption"; import PATHS from "router/paths"; @@ -30,6 +34,7 @@ const DEFAULT_ACTION_OPTIONS: IDropdownOption[] = [ type ISoftwareTableConfig = Column; type ITableHeaderProps = IHeaderProps; +type ITableNumberCellProps = INumberCellProps; type ITableStringCellProps = IStringCellProps; type IInstalledStatusCellProps = CellProps< IHostSoftware, @@ -43,18 +48,23 @@ type IVulnerabilitiesCellProps = IInstalledVersionsCellProps; // type IActionsCellProps = CellProps; const generateActions = ( - packageToInstall: string | null, softwareId: number, - installingSoftwareId: number | null + status: ISoftwareInstallStatus | null, + installingSoftwareId: number | null, + canInstall: boolean, + packageToInstall?: string | null ) => { // this gives us a clean slate of the default actions so we can modify // the options. - const actions = cloneDeep(DEFAULT_ACTION_OPTIONS); + let actions = cloneDeep(DEFAULT_ACTION_OPTIONS); - // TODO: when do we not show the options? + // remove install if there is no package to install + if (!packageToInstall || !canInstall) { + actions = actions.filter((action) => action.value !== "install"); + } // disable install option if software is already installing - if (softwareId === installingSoftwareId) { + if (softwareId === installingSoftwareId || status === "pending") { const installAction = actions.find((action) => action.value === "install"); if (installAction) { installAction.disabled = true; @@ -67,6 +77,7 @@ const generateActions = ( interface ISoftwareTableHeadersProps { installingSoftwareId: number | null; onSelectAction: (software: IHostSoftware, action: string) => void; + canInstall: boolean; router: InjectedRouter; } @@ -76,6 +87,7 @@ export const generateSoftwareTableHeaders = ({ router, installingSoftwareId, onSelectAction, + canInstall, }: ISoftwareTableHeadersProps): ISoftwareTableConfig[] => { const tableHeaders: ISoftwareTableConfig[] = [ { @@ -108,13 +120,13 @@ export const generateSoftwareTableHeaders = ({ Cell: (cellProps: IInstalledStatusCellProps) => { const { original } = cellProps.row; const { value } = cellProps.cell; - return value ? ( + return ( - ) : null; + ); }, }, { @@ -153,14 +165,16 @@ export const generateSoftwareTableHeaders = ({ disableSortBy: true, // the accessor here is insignificant, we just need it as its required // but we don't use it. - accessor: "bundle_identifier", - Cell: (cellProps: ITableStringCellProps) => ( + accessor: "id", + Cell: (cellProps: ITableNumberCellProps) => ( onSelectAction(cellProps.row.original, action)} /> diff --git a/frontend/pages/hosts/details/cards/Software/InstallStatusCell/InstallStatusCell.tsx b/frontend/pages/hosts/details/cards/Software/InstallStatusCell/InstallStatusCell.tsx index 0ca7d9214c..c83d154ff2 100644 --- a/frontend/pages/hosts/details/cards/Software/InstallStatusCell/InstallStatusCell.tsx +++ b/frontend/pages/hosts/details/cards/Software/InstallStatusCell/InstallStatusCell.tsx @@ -5,6 +5,7 @@ import { dateAgo } from "utilities/date_format"; import Icon from "components/Icon"; import TooltipWrapper from "components/TooltipWrapper"; +import TextCell from "components/TableContainer/DataTable/TextCell"; const baseClass = "install-status-cell"; @@ -13,7 +14,7 @@ type IStatusValue = ISoftwareInstallStatus | "avaiableForInstall"; type IStatusDisplayConfig = { iconName: "success" | "pending-outline" | "error" | "install"; displayText: string; - tooltip: (softwareName: string | null, lastInstall?: string) => ReactNode; + tooltip: (softwareName?: string | null, lastInstall?: string) => ReactNode; }; const CELL_DISPLAY_OPTIONS: Record = { @@ -56,7 +57,7 @@ const CELL_DISPLAY_OPTIONS: Record = { interface IInstallStatusCellProps { status: ISoftwareInstallStatus | null; - packageToInstall: string | null; + packageToInstall?: string | null; installedAt?: string; } @@ -66,10 +67,13 @@ const InstallStatusCell = ({ installedAt, }: IInstallStatusCellProps) => { let displayStatus: IStatusValue; - if (status === null) { + + if (packageToInstall) { displayStatus = "avaiableForInstall"; - } else { + } else if (status !== null) { displayStatus = status; + } else { + return ; } const displayConfig = CELL_DISPLAY_OPTIONS[displayStatus]; @@ -79,6 +83,7 @@ const InstallStatusCell = ({ tipContent={displayConfig.tooltip(packageToInstall, installedAt)} underline={false} className={baseClass} + tooltipClass={`${baseClass}__status-tooltip`} position="top" >
diff --git a/frontend/pages/hosts/details/cards/Software/InstallStatusCell/_styles.scss b/frontend/pages/hosts/details/cards/Software/InstallStatusCell/_styles.scss index cadd24df28..10f8b9c53a 100644 --- a/frontend/pages/hosts/details/cards/Software/InstallStatusCell/_styles.scss +++ b/frontend/pages/hosts/details/cards/Software/InstallStatusCell/_styles.scss @@ -4,4 +4,8 @@ align-items: center; gap: $pad-small; } + + &__status-tooltip { + text-align: center; + } } diff --git a/frontend/pages/hosts/details/cards/Software/Software.tsx b/frontend/pages/hosts/details/cards/Software/Software.tsx index 021a3eafd5..70641a41e0 100644 --- a/frontend/pages/hosts/details/cards/Software/Software.tsx +++ b/frontend/pages/hosts/details/cards/Software/Software.tsx @@ -3,19 +3,25 @@ import { InjectedRouter } from "react-router"; import { useQuery } from "react-query"; import { AxiosError } from "axios"; -import hostAPI, { IGetHostSoftwareResponse } from "services/entities/hosts"; +import hostAPI, { + IGetHostSoftwareResponse, + IHostSoftwareQueryParams, +} from "services/entities/hosts"; import deviceAPI, { + IDeviceSoftwareQueryParams, IGetDeviceSoftwareResponse, } from "services/entities/device_user"; import { IHostSoftware, ISoftware } from "interfaces/software"; import { DEFAULT_USE_QUERY_OPTIONS } from "utilities/constants"; import { NotificationContext } from "context/notification"; +import { AppContext } from "context/app"; import Card from "components/Card"; import Spinner from "components/Spinner"; import DataError from "components/DataError"; -import { generateSoftwareTableHeaders } from "./HostSoftwareTableConfig"; +import { generateSoftwareTableHeaders as generateHostSoftwareTableConfig } from "./HostSoftwareTableConfig"; +import { generateSoftwareTableHeaders as generateDeviceSoftwareTableConfig } from "./DeviceSoftwareTableConfig"; import HostSoftwareTable from "./HostSoftwareTable"; const baseClass = "software-card"; @@ -44,6 +50,7 @@ const DEFAULT_SEARCH_QUERY = ""; const DEFAULT_SORT_DIRECTION = "desc"; const DEFAULT_SORT_HEADER = "name"; const DEFAULT_PAGE = 0; +const DEFAULT_PAGE_SIZE = 20; const SoftwareCard = ({ id, @@ -55,19 +62,49 @@ const SoftwareCard = ({ isMyDevicePage = false, }: ISoftwareCardProps) => { const { renderFlash } = useContext(NotificationContext); + const { + isGlobalAdmin, + isGlobalMaintainer, + isTeamAdmin, + isTeamMaintainer, + } = useContext(AppContext); const [installingSoftwareId, setInstallingSoftwareId] = useState< number | null >(null); + const searchQuery = queryParams?.query ?? DEFAULT_SEARCH_QUERY; + const sortHeader = queryParams?.order_key ?? DEFAULT_SORT_HEADER; + const sortDirection = queryParams?.order_direction ?? DEFAULT_SORT_DIRECTION; + const page = queryParams?.page + ? parseInt(queryParams.page, 10) + : DEFAULT_PAGE; + const pageSize = DEFAULT_PAGE_SIZE; + const { data: hostSoftwareRes, isLoading: hostSoftwareLoading, isError: hostSoftwareError, isFetching: hostSoftwareFetching, - } = useQuery( - ["host-software", queryParams], - () => hostAPI.getHostSoftware(id as number), + } = useQuery< + IGetHostSoftwareResponse, + AxiosError, + IGetHostSoftwareResponse, + [string, IHostSoftwareQueryParams] + >( + [ + "host-software", + { + page, + per_page: pageSize, + query: searchQuery, + order_key: sortHeader, + order_direction: sortDirection, + }, + ], + ({ queryKey }) => { + return hostAPI.getHostSoftware(id as number, queryKey[1]); + }, { ...DEFAULT_USE_QUERY_OPTIONS, enabled: isSoftwareEnabled && !isMyDevicePage, @@ -79,21 +116,32 @@ const SoftwareCard = ({ isLoading: deviceSoftwareLoading, isError: deviceSoftwareError, isFetching: deviceSoftwareFetching, - } = useQuery( - ["host-software", queryParams], - () => deviceAPI.getDeviceSoftware(id as string), + } = useQuery< + IGetDeviceSoftwareResponse, + AxiosError, + IGetDeviceSoftwareResponse, + [string, IDeviceSoftwareQueryParams] + >( + [ + "device-software", + { + page, + per_page: pageSize, + query: searchQuery, + order_key: sortHeader, + order_direction: sortDirection, + }, + ], + ({ queryKey }) => deviceAPI.getDeviceSoftware(id as string, queryKey[1]), { ...DEFAULT_USE_QUERY_OPTIONS, enabled: isSoftwareEnabled && isMyDevicePage, } ); - const searchQuery = queryParams?.query ?? DEFAULT_SEARCH_QUERY; - const sortHeader = queryParams?.order_key ?? DEFAULT_SORT_HEADER; - const sortDirection = queryParams?.order_direction ?? DEFAULT_SORT_DIRECTION; - const page = queryParams?.page - ? parseInt(queryParams.page, 10) - : DEFAULT_PAGE; + const canInstallSoftware = Boolean( + isGlobalAdmin || isGlobalMaintainer || isTeamAdmin || isTeamMaintainer + ); const installHostSoftwarePackage = useCallback( async (softwareId: number) => { @@ -128,13 +176,22 @@ const SoftwareCard = ({ [installHostSoftwarePackage, onShowSoftwareDetails] ); - const tableHeaders = useMemo(() => { - return generateSoftwareTableHeaders({ - router, - installingSoftwareId, - onSelectAction, - }); - }, [installingSoftwareId, router, onSelectAction]); + const tableConfig = useMemo(() => { + return isMyDevicePage + ? generateDeviceSoftwareTableConfig() + : generateHostSoftwareTableConfig({ + router, + installingSoftwareId, + canInstall: canInstallSoftware, + onSelectAction, + }); + }, [ + isMyDevicePage, + router, + installingSoftwareId, + canInstallSoftware, + onSelectAction, + ]); const renderSoftwareTable = () => { if (hostSoftwareLoading || deviceSoftwareLoading) { @@ -145,23 +202,33 @@ const SoftwareCard = ({ return ; } - if (!hostSoftwareRes || !deviceSoftwareRes) { - return null; + const props = { + router, + tableConfig, + sortHeader, + sortDirection, + searchQuery, + page, + pagePath: pathname, + }; + + if (!isMyDevicePage) { + return hostSoftwareRes ? ( + + ) : null; } - return ( + return deviceSoftwareRes ? ( - ); + ) : null; }; return ( diff --git a/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/SoftwareDetailsModal.tsx b/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/SoftwareDetailsModal.tsx index 372d5f3f7a..6dd73df85e 100644 --- a/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/SoftwareDetailsModal.tsx +++ b/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/SoftwareDetailsModal.tsx @@ -32,7 +32,7 @@ const generateVulnerabilitiesValue = (vulnerabilities: string[]) => { interface ISoftwareDetailsInfoProps { installedVersion: ISoftwareInstallVersion; source: string; - bundleIdentifier: string; + bundleIdentifier?: string; } const SoftwareDetailsInfo = ({ @@ -40,12 +40,16 @@ const SoftwareDetailsInfo = ({ source, bundleIdentifier, }: ISoftwareDetailsInfoProps) => { + const { vulnerabilities } = installedVersion; + return (
- + {bundleIdentifier && ( + + )} {installedVersion.last_opened_at && ( ( - <>{path} - ))} - /> -
-
- + {installedVersion.installed_paths.map((path) => ( + {path} + ))} +
+ } />
+ {vulnerabilities && vulnerabilities.length !== 0 && ( +
+ +
+ )}
); }; @@ -81,38 +91,54 @@ const SoftwareDetailsModal = ({ onExit, }: ISoftwareDetailsModalProps) => { const renderSoftwareDetails = () => { - if ( - !software.installed_versions || - software.installed_versions.length === 0 - ) { - return null; - } + const { installed_versions } = software; - return software.installed_versions.map((installedVersion) => { + // special case when we dont have installed versions. We can only show the + // software type atm. + if (!installed_versions || installed_versions.length === 0) { return ( - ); - }); + } + + return ( +
+ {installed_versions.map((installedVersion) => { + return ( + + ); + })} +
+ ); + }; + + const renderTabs = () => { + return ( + + + + Software details + Install Details + + {renderSoftwareDetails()} + test 2 + + + ); }; return ( <> - - - - Software details - Install Details - - {renderSoftwareDetails()} - test 2 - - + {software.last_install ? renderTabs() : renderSoftwareDetails()}
diff --git a/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/_styles.scss b/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/_styles.scss index 6467b48c6d..6b241115a0 100644 --- a/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/_styles.scss +++ b/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/_styles.scss @@ -22,9 +22,25 @@ gap: $pad-xxlarge; } + &__file-path-data-set { + // These following 100% widths are to make sure the text does not + // overflow from the modal. TODO: Need to look at DataSet component to see why + // it overflows. + width: 100%; + min-width: auto; + } + &__file-path-values { display: flex; flex-direction: column; gap: $pad-small; + width: 100%; + + > span { + width: 100%; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } } } diff --git a/frontend/services/entities/software.ts b/frontend/services/entities/software.ts index 778af718ea..b81539a288 100644 --- a/frontend/services/entities/software.ts +++ b/frontend/services/entities/software.ts @@ -205,17 +205,10 @@ export default { return sendRequest("POST", SOFTWARE_PACKAGE_ADD, formData); }, - deleteSoftwarePackage: (softwareId: number) => { + deleteSoftwarePackage: (softwareId: number, teamId: number) => { const { SOFTWARE_PACKAGE } = endpoints; - return sendRequest("DELETE", SOFTWARE_PACKAGE(softwareId)); - }, - - downloadSoftwarePackage: (softwareId: number) => { - const { SOFTWARE_PACKAGE } = endpoints; - const path = `${SOFTWARE_PACKAGE(softwareId)}?${buildQueryStringFromParams({ - alt: "media", - })}`; - return sendRequest("GET", path); + const path = `${SOFTWARE_PACKAGE(softwareId)}?team_id=${teamId}`; + return sendRequest("DELETE", path); }, getSoftwareInstallResult: (installUuid: string) => { diff --git a/frontend/utilities/endpoints.ts b/frontend/utilities/endpoints.ts index b0d4d9c7da..1741b05ea6 100644 --- a/frontend/utilities/endpoints.ts +++ b/frontend/utilities/endpoints.ts @@ -140,7 +140,7 @@ export default { `/${API_VERSION}/fleet/software/versions/${id}`, SOFTWARE_PACKAGE_ADD: `/${API_VERSION}/fleet/software/package`, SOFTWARE_PACKAGE: (id: number) => - `/${API_VERSION}/fleet/software/packages/${id}`, + `/${API_VERSION}/fleet/software/${id}/package`, SOFTWARE_INSTALL_RESULTS: (uuid: string) => `/${API_VERSION}/fleet/software/install/results/${uuid}`, SOFTWARE_PACKAGE_INSTALL: (id: number) => diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 6a21e1801d..8f13b5547a 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -184,18 +184,20 @@ WHERE func (ds *Datastore) GetSoftwareInstallerMetadataByTeamAndTitleID(ctx context.Context, teamID *uint, titleID uint) (*fleet.SoftwareInstaller, error) { query := ` SELECT - id, - team_id, - title_id, - storage_id, - filename, - version, - install_script_content_id, - pre_install_query, - post_install_script_content_id, - uploaded_at + si.id, + si.team_id, + si.title_id, + si.storage_id, + si.filename, + si.version, + si.install_script_content_id, + si.pre_install_query, + si.post_install_script_content_id, + si.uploaded_at, + COALESCE(st.name, '') AS software_title FROM - software_installers + software_installers si + LEFT OUTER JOIN software_titles st ON st.id = si.title_id WHERE title_id = ? AND global_or_team_id = ?` diff --git a/server/datastore/mysql/software_titles.go b/server/datastore/mysql/software_titles.go index d8c47f850b..273cbed594 100644 --- a/server/datastore/mysql/software_titles.go +++ b/server/datastore/mysql/software_titles.go @@ -83,6 +83,10 @@ func (ds *Datastore) ListSoftwareTitles( opt.ListOptions.OrderDirection = fleet.OrderDescending } + if opt.AvailableForInstall && opt.VulnerableOnly { + return nil, 0, nil, fleet.NewInvalidArgumentError("query", "available_for_install and vulnerable can't be provided together") + } + dbReader := ds.reader(ctx) getTitlesStmt, args := selectSoftwareTitlesSQL(opt) // build the count statement before adding the pagination constraints to `getTitlesStmt` @@ -206,10 +210,8 @@ LEFT JOIN software_titles_host_counts sthc ON sthc.software_title_id = st.id AND %s -- placeholder for optional extra WHERE filter WHERE %s -AND ( - sthc.hosts_count > 0 OR - EXISTS (SELECT 1 FROM software_installers si WHERE si.title_id = st.id AND si.global_or_team_id = ?) -) +-- placeholder for filter based on software installed on hosts + software installers +AND (%s) GROUP BY st.id` cveJoinType := "LEFT" @@ -244,9 +246,29 @@ GROUP BY st.id` match = likePattern(match) args = append(args, match, match) } + + defaultFilter := ` + EXISTS ( + SELECT 1 + FROM + software_installers si + WHERE + si.title_id = st.id + AND si.global_or_team_id = ? + ) + ` + + // add software installed for hosts if any of this is true: + // + // - we're not filtering for "available for install" only + // - we're filtering by vulnerable only + if !opt.AvailableForInstall || opt.VulnerableOnly { + defaultFilter += `OR sthc.hosts_count > 0` + } + args = append(args, globalOrTeamID) - stmt = fmt.Sprintf(stmt, softwareJoin, additionalWhere) + stmt = fmt.Sprintf(stmt, softwareJoin, additionalWhere, defaultFilter) return stmt, args } diff --git a/server/datastore/mysql/software_titles_test.go b/server/datastore/mysql/software_titles_test.go index 7fdd4cf5a5..c4dffd9a63 100644 --- a/server/datastore/mysql/software_titles_test.go +++ b/server/datastore/mysql/software_titles_test.go @@ -26,6 +26,7 @@ func TestSoftwareTitles(t *testing.T) { {"OrderSoftwareTitles", testOrderSoftwareTitles}, {"TeamFilterSoftwareTitles", testTeamFilterSoftwareTitles}, {"ListSoftwareTitlesInstallersOnly", testListSoftwareTitlesInstallersOnly}, + {"ListSoftwareTitlesAvailableForInstallFilter", testListSoftwareTitlesAvailableForInstallFilter}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -77,7 +78,7 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) globalOpts := fleet.SoftwareTitleListOptions{ListOptions: fleet.ListOptions{OrderKey: "hosts_count", OrderDirection: fleet.OrderDescending}} - globalCounts := listSoftwareTitlesCheckCount(t, ds, 2, 2, globalOpts, false) + globalCounts := listSoftwareTitlesCheckCount(t, ds, 2, 2, globalOpts) want := []fleet.SoftwareTitle{ {Name: "foo", HostsCount: 2}, @@ -97,7 +98,7 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) - globalCounts = listSoftwareTitlesCheckCount(t, ds, 1, 1, globalOpts, false) + globalCounts = listSoftwareTitlesCheckCount(t, ds, 1, 1, globalOpts) want = []fleet.SoftwareTitle{ {Name: "foo", HostsCount: 2}, } @@ -109,7 +110,7 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { require.NoError(t, err) // listing does not return the new software title entry - allSw := listSoftwareTitlesCheckCount(t, ds, 1, 1, fleet.SoftwareTitleListOptions{}, false) + allSw := listSoftwareTitlesCheckCount(t, ds, 1, 1, fleet.SoftwareTitleListOptions{}) want = []fleet.SoftwareTitle{ {Name: "foo", HostsCount: 2}, } @@ -142,7 +143,7 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { require.NoError(t, err) // at this point, there's no counts per team, only global counts - globalCounts = listSoftwareTitlesCheckCount(t, ds, 1, 1, globalOpts, false) + globalCounts = listSoftwareTitlesCheckCount(t, ds, 1, 1, globalOpts) want = []fleet.SoftwareTitle{ {Name: "foo", HostsCount: 2}, } @@ -153,7 +154,7 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { TeamID: ptr.Uint(team1.ID), ListOptions: fleet.ListOptions{OrderKey: "hosts_count", OrderDirection: fleet.OrderDescending}, } - team1Counts := listSoftwareTitlesCheckCount(t, ds, 0, 0, team1Opts, false) + team1Counts := listSoftwareTitlesCheckCount(t, ds, 0, 0, team1Opts) want = []fleet.SoftwareTitle{} cmpNameVersionCount(want, team1Counts) checkTableTotalCount(1) @@ -163,14 +164,14 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) - globalCounts = listSoftwareTitlesCheckCount(t, ds, 2, 2, globalOpts, false) + globalCounts = listSoftwareTitlesCheckCount(t, ds, 2, 2, globalOpts) want = []fleet.SoftwareTitle{ {Name: "foo", HostsCount: 4}, {Name: "bar", HostsCount: 1}, } cmpNameVersionCount(want, globalCounts) - team1Counts = listSoftwareTitlesCheckCount(t, ds, 1, 1, team1Opts, false) + team1Counts = listSoftwareTitlesCheckCount(t, ds, 1, 1, team1Opts) want = []fleet.SoftwareTitle{ {Name: "foo", HostsCount: 2}, } @@ -183,7 +184,7 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { TeamID: ptr.Uint(team2.ID), ListOptions: fleet.ListOptions{OrderKey: "hosts_count", OrderDirection: fleet.OrderDescending}, } - team2Counts := listSoftwareTitlesCheckCount(t, ds, 2, 2, team2Opts, false) + team2Counts := listSoftwareTitlesCheckCount(t, ds, 2, 2, team2Opts) want = []fleet.SoftwareTitle{ {Name: "foo", HostsCount: 1}, {Name: "bar", HostsCount: 1}, @@ -201,19 +202,19 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) - globalCounts = listSoftwareTitlesCheckCount(t, ds, 1, 1, globalOpts, false) + globalCounts = listSoftwareTitlesCheckCount(t, ds, 1, 1, globalOpts) want = []fleet.SoftwareTitle{ {Name: "foo", HostsCount: 4}, } cmpNameVersionCount(want, globalCounts) - team1Counts = listSoftwareTitlesCheckCount(t, ds, 1, 1, team1Opts, false) + team1Counts = listSoftwareTitlesCheckCount(t, ds, 1, 1, team1Opts) want = []fleet.SoftwareTitle{ {Name: "foo", HostsCount: 2}, } cmpNameVersionCount(want, team1Counts) - team2Counts = listSoftwareTitlesCheckCount(t, ds, 1, 1, team2Opts, false) + team2Counts = listSoftwareTitlesCheckCount(t, ds, 1, 1, team2Opts) want = []fleet.SoftwareTitle{ {Name: "foo", HostsCount: 1}, } @@ -228,7 +229,7 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now())) require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) - listSoftwareTitlesCheckCount(t, ds, 0, 0, team2Opts, false) + listSoftwareTitlesCheckCount(t, ds, 0, 0, team2Opts) // delete team require.NoError(t, ds.DeleteTeam(ctx, team2.ID)) @@ -238,19 +239,19 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) - globalCounts = listSoftwareTitlesCheckCount(t, ds, 1, 1, globalOpts, false) + globalCounts = listSoftwareTitlesCheckCount(t, ds, 1, 1, globalOpts) want = []fleet.SoftwareTitle{ {Name: "foo", HostsCount: 3}, } cmpNameVersionCount(want, globalCounts) - team1Counts = listSoftwareTitlesCheckCount(t, ds, 1, 1, team1Opts, false) + team1Counts = listSoftwareTitlesCheckCount(t, ds, 1, 1, team1Opts) want = []fleet.SoftwareTitle{ {Name: "foo", HostsCount: 2}, } cmpNameVersionCount(want, team1Counts) - listSoftwareTitlesCheckCount(t, ds, 0, 0, team2Opts, false) + listSoftwareTitlesCheckCount(t, ds, 0, 0, team2Opts) checkTableTotalCount(2) } @@ -456,7 +457,7 @@ func testOrderSoftwareTitles(t *testing.T, ds *Datastore) { require.Equal(t, "apps", titles[1].Source) } -func listSoftwareTitlesCheckCount(t *testing.T, ds *Datastore, expectedListCount int, expectedFullCount int, opts fleet.SoftwareTitleListOptions, returnSorted bool) []fleet.SoftwareTitle { +func listSoftwareTitlesCheckCount(t *testing.T, ds *Datastore, expectedListCount int, expectedFullCount int, opts fleet.SoftwareTitleListOptions) []fleet.SoftwareTitle { titles, count, _, err := ds.ListSoftwareTitles(context.Background(), opts, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}}) require.NoError(t, err) require.Len(t, titles, expectedListCount) @@ -670,4 +671,86 @@ func testListSoftwareTitlesInstallersOnly(t *testing.T, ds *Datastore) { require.NoError(t, err) require.EqualValues(t, 0, counts) require.Len(t, titles, 0) + + // using the available_for_install filter + titles, counts, _, err = ds.ListSoftwareTitles( + ctx, + fleet.SoftwareTitleListOptions{ + ListOptions: fleet.ListOptions{ + OrderKey: "name", + OrderDirection: fleet.OrderAscending, + }, + AvailableForInstall: true, + }, + fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}}, + ) + require.NoError(t, err) + require.EqualValues(t, 2, counts) + require.Len(t, titles, 2) + require.True(t, titles[0].CountsUpdatedAt.IsZero()) + +} + +func testListSoftwareTitlesAvailableForInstallFilter(t *testing.T, ds *Datastore) { + ctx := context.Background() + + installer1, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + Title: "installer1", + Source: "apps", + InstallScript: "echo", + Filename: "installer1.pkg", + }) + require.NoError(t, err) + require.NotZero(t, installer1) + installer2, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + Title: "installer2", + Source: "apps", + InstallScript: "echo", + Filename: "installer2.pkg", + }) + require.NoError(t, err) + require.NotZero(t, installer2) + + host := test.NewHost(t, ds, "host", "", "hostkey", "hostuuid", time.Now()) + software := []fleet.Software{ + {Name: "foo", Version: "0.0.1", Source: "chrome_extensions"}, + {Name: "foo", Version: "0.0.3", Source: "chrome_extensions"}, + {Name: "bar", Version: "0.0.3", Source: "deb_packages"}, + } + _, err = ds.UpdateHostSoftware(ctx, host.ID, software) + require.NoError(t, err) + require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now())) + require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) + require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) + + // without filter returns all software + titles, counts, _, err := ds.ListSoftwareTitles( + ctx, + fleet.SoftwareTitleListOptions{ + ListOptions: fleet.ListOptions{ + OrderKey: "name", + OrderDirection: fleet.OrderAscending, + }, + }, + fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}}, + ) + require.NoError(t, err) + require.EqualValues(t, 4, counts) + require.Len(t, titles, 4) + + // with filter returns only available for install + titles, counts, _, err = ds.ListSoftwareTitles( + ctx, + fleet.SoftwareTitleListOptions{ + ListOptions: fleet.ListOptions{ + OrderKey: "name", + OrderDirection: fleet.OrderAscending, + }, + AvailableForInstall: true, + }, + fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}}, + ) + require.NoError(t, err) + require.EqualValues(t, 2, counts) + require.Len(t, titles, 2) } diff --git a/server/fleet/service.go b/server/fleet/service.go index ae77d8a176..50f3922532 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -1020,8 +1020,8 @@ type Service interface { // UploadSoftwareInstaller(ctx context.Context, payload *UploadSoftwareInstallerPayload) error - DeleteSoftwareInstaller(ctx context.Context, id uint) error - GetSoftwareInstallerMetadata(ctx context.Context, id uint) (*SoftwareInstaller, error) - DownloadSoftwareInstaller(ctx context.Context, installerID uint) (*DownloadSoftwareInstallerPayload, error) + DeleteSoftwareInstaller(ctx context.Context, titleID uint, teamID *uint) error + GetSoftwareInstallerMetadata(ctx context.Context, titleID uint, teamID *uint) (*SoftwareInstaller, error) + DownloadSoftwareInstaller(ctx context.Context, titleID uint, teamID *uint) (*DownloadSoftwareInstallerPayload, error) OrbitDownloadSoftwareInstaller(ctx context.Context, installerID uint) (*DownloadSoftwareInstallerPayload, error) } diff --git a/server/fleet/software.go b/server/fleet/software.go index d5afe4deab..4ab963b75b 100644 --- a/server/fleet/software.go +++ b/server/fleet/software.go @@ -166,8 +166,9 @@ type SoftwareTitleListOptions struct { // ListOptions cannot be embedded in order to unmarshall with validation. ListOptions ListOptions `url:"list_options"` - TeamID *uint `query:"team_id,optional"` - VulnerableOnly bool `query:"vulnerable,optional"` + TeamID *uint `query:"team_id,optional"` + VulnerableOnly bool `query:"vulnerable,optional"` + AvailableForInstall bool `query:"available_for_install,optional"` } // AuthzSoftwareInventory is used for access controls on software inventory. diff --git a/server/service/handler.go b/server/service/handler.go index 01969c4b8a..e358041844 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -373,9 +373,9 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC ue.POST("/api/v1/fleet/hosts/{host_id:[0-9]+}/software/install/{software_title_id:[0-9]+}", installSoftwareTitleEndpoint, installSoftwareRequest{}) // Sofware installers - ue.GET("/api/_version_/fleet/software/package/{id:[0-9]+}", getSoftwareInstallerEndpoint, getSoftwareInstallerRequest{}) + ue.GET("/api/_version_/fleet/software/{title_id:[0-9]+}/package", getSoftwareInstallerEndpoint, getSoftwareInstallerRequest{}) ue.POST("/api/_version_/fleet/software/package", uploadSoftwareInstallerEndpoint, uploadSoftwareInstallerRequest{}) - ue.DELETE("/api/_version_/fleet/software/package/{id:[0-9]+}", deleteSoftwareInstallerEndpoint, deleteSoftwareInstallerRequest{}) + ue.DELETE("/api/_version_/fleet/software/{title_id:[0-9]+}/package", deleteSoftwareInstallerEndpoint, deleteSoftwareInstallerRequest{}) ue.GET("/api/_version_/fleet/software/install/results/{install_uuid}", getSoftwareInstallResultsEndpoint, getSoftwareInstallResultsRequest{}) // Vulnerabilities diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index bd4cdb2050..9e8201b099 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -7168,7 +7168,7 @@ func (s *integrationEnterpriseTestSuite) TestAllSoftwareTitles() { ) require.Equal(t, 2, resp.Count) require.Empty(t, resp.CountsUpdatedAt) - softwareTitlesMatch(nil, resp.SoftwareTitles) + softwareTitlesMatch([]fleet.SoftwareTitle{}, resp.SoftwareTitles) // asking for vulnerable only software returns the expected values resp = listSoftwareTitlesResponse{} @@ -7202,7 +7202,7 @@ func (s *integrationEnterpriseTestSuite) TestAllSoftwareTitles() { ) require.Equal(t, 0, resp.Count) require.Empty(t, resp.CountsUpdatedAt) - softwareTitlesMatch(nil, resp.SoftwareTitles) + softwareTitlesMatch([]fleet.SoftwareTitle{}, resp.SoftwareTitles) // add new software for tmHost software = []fleet.Software{ diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 2d4c330178..8e415a10ff 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -8516,7 +8516,7 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadDownloadAndDelete() require.Equal(t, expectedContents, contents) } - checkSoftwareInstaller := func(t *testing.T, payload *fleet.UploadSoftwareInstallerPayload) uint { + checkSoftwareInstaller := func(t *testing.T, payload *fleet.UploadSoftwareInstallerPayload) (installerID uint, titleID uint) { var id uint mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { var tid uint @@ -8552,7 +8552,7 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadDownloadAndDelete() require.Equal(t, checkSoftwareTitle(t, payload.Title, "deb_packages"), *meta.TitleID) require.NotZero(t, meta.UploadedAt) - return meta.InstallerID + return meta.InstallerID, *meta.TitleID } t.Run("upload no team software installer", func(t *testing.T) { @@ -8574,28 +8574,16 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadDownloadAndDelete() s.lastActivityOfTypeMatches(fleet.ActivityTypeAddedSoftware{}.ActivityName(), `{"software_title": "ruby", "software_package": "ruby.deb", "team_name": null, "team_id": null}`, 0) // check the software installer - installerID := checkSoftwareInstaller(t, payload) + _, titleID := checkSoftwareInstaller(t, payload) // upload again fails s.uploadSoftwareInstaller(payload, http.StatusConflict, "already exists") // download the installer - r := s.Do("GET", fmt.Sprintf("/api/latest/fleet/software/package/%d?alt=media", installerID), nil, http.StatusOK) - checkDownloadResponse(t, r, payload.Filename) - - // create an orbit host and request to download the installer - host := createOrbitEnrolledHost(t, "windows", "orbit-host", s.ds) - r = s.Do("POST", "/api/fleet/orbit/software_install/package?alt=media", orbitDownloadSoftwareInstallerRequest{ - InstallerID: installerID, - OrbitNodeKey: *host.OrbitNodeKey, - }, http.StatusOK) - checkDownloadResponse(t, r, payload.Filename) + s.Do("GET", fmt.Sprintf("/api/latest/fleet/software/%d/package?alt=media", titleID), nil, http.StatusBadRequest) // delete the installer - s.Do("DELETE", fmt.Sprintf("/api/latest/fleet/software/package/%d", installerID), nil, http.StatusNoContent) - - // check activity - s.lastActivityOfTypeMatches(fleet.ActivityTypeDeletedSoftware{}.ActivityName(), `{"software_title": "ruby", "software_package": "ruby.deb", "team_name": null, "team_id": null}`, 0) + s.Do("DELETE", fmt.Sprintf("/api/latest/fleet/software/%d/package", titleID), nil, http.StatusBadRequest) }) t.Run("create team software installer", func(t *testing.T) { @@ -8621,7 +8609,7 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadDownloadAndDelete() s.uploadSoftwareInstaller(payload, http.StatusOK, "") // check the software installer - installerID := checkSoftwareInstaller(t, payload) + installerID, titleID := checkSoftwareInstaller(t, payload) // check activity s.lastActivityOfTypeMatches(fleet.ActivityTypeAddedSoftware{}.ActivityName(), fmt.Sprintf(`{"software_title": "ruby", "software_package": "ruby.deb", "team_name": "%s", "team_id": %d}`, createTeamResp.Team.Name, createTeamResp.Team.ID), 0) @@ -8630,7 +8618,7 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadDownloadAndDelete() s.uploadSoftwareInstaller(payload, http.StatusConflict, "already exists") // download the installer - r := s.Do("GET", fmt.Sprintf("/api/latest/fleet/software/package/%d?alt=media", installerID), nil, http.StatusOK) + r := s.Do("GET", fmt.Sprintf("/api/latest/fleet/software/%d/package?alt=media", titleID), nil, http.StatusOK, "team_id", fmt.Sprintf("%d", *payload.TeamID)) checkDownloadResponse(t, r, payload.Filename) // create an orbit host, assign to team and request to download the installer @@ -8643,7 +8631,7 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadDownloadAndDelete() checkDownloadResponse(t, r, payload.Filename) // delete the installer - s.Do("DELETE", fmt.Sprintf("/api/latest/fleet/software/package/%d", installerID), nil, http.StatusNoContent) + s.Do("DELETE", fmt.Sprintf("/api/latest/fleet/software/%d/package", titleID), nil, http.StatusNoContent, "team_id", fmt.Sprintf("%d", *payload.TeamID)) // check activity s.lastActivityOfTypeMatches(fleet.ActivityTypeDeletedSoftware{}.ActivityName(), fmt.Sprintf(`{"software_title": "ruby", "software_package": "ruby.deb", "team_name": "%s", "team_id": %d}`, createTeamResp.Team.Name, createTeamResp.Team.ID), 0) diff --git a/server/service/software_installers.go b/server/service/software_installers.go index 1597eb9ec0..d2528401fe 100644 --- a/server/service/software_installers.go +++ b/server/service/software_installers.go @@ -116,7 +116,8 @@ func (svc *Service) UploadSoftwareInstaller(ctx context.Context, payload *fleet. } type deleteSoftwareInstallerRequest struct { - ID uint `url:"id"` + TeamID *uint `query:"team_id"` + TitleID uint `url:"title_id"` } type deleteSoftwareInstallerResponse struct { @@ -128,14 +129,14 @@ func (r deleteSoftwareInstallerResponse) Status() int { return http.StatusNoCon func deleteSoftwareInstallerEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { req := request.(*deleteSoftwareInstallerRequest) - err := svc.DeleteSoftwareInstaller(ctx, req.ID) + err := svc.DeleteSoftwareInstaller(ctx, req.TitleID, req.TeamID) if err != nil { return deleteSoftwareInstallerResponse{Err: err}, nil } return deleteSoftwareInstallerResponse{}, nil } -func (svc *Service) DeleteSoftwareInstaller(ctx context.Context, id uint) error { +func (svc *Service) DeleteSoftwareInstaller(ctx context.Context, titleID uint, teamID *uint) error { // skipauth: No authorization check needed due to implementation returning // only license error. svc.authz.SkipAuthorization(ctx) @@ -144,8 +145,9 @@ func (svc *Service) DeleteSoftwareInstaller(ctx context.Context, id uint) error } type getSoftwareInstallerRequest struct { - Alt string `query:"alt,optional"` - InstallerID uint `url:"id"` + Alt string `query:"alt,optional"` + TeamID *uint `query:"team_id"` + TitleID uint `url:"title_id"` } type getSoftwareInstallerResponse struct { @@ -164,7 +166,7 @@ func getSoftwareInstallerEndpoint(ctx context.Context, request interface{}, svc return getSoftwareInstallerResponse{Err: &fleet.BadRequestError{Message: "only alt=media is supported"}}, nil } - payload, err := svc.DownloadSoftwareInstaller(ctx, req.InstallerID) + payload, err := svc.DownloadSoftwareInstaller(ctx, req.TitleID, req.TeamID) if err != nil { return downloadSoftwareInstallerResponse{Err: err}, nil } @@ -172,7 +174,7 @@ func getSoftwareInstallerEndpoint(ctx context.Context, request interface{}, svc return downloadSoftwareInstallerResponse{payload: payload}, nil } -func (svc *Service) GetSoftwareInstallerMetadata(ctx context.Context, id uint) (*fleet.SoftwareInstaller, error) { +func (svc *Service) GetSoftwareInstallerMetadata(ctx context.Context, titleID uint, teamID *uint) (*fleet.SoftwareInstaller, error) { // skipauth: No authorization check needed due to implementation returning // only license error. svc.authz.SkipAuthorization(ctx) @@ -203,7 +205,7 @@ func (r downloadSoftwareInstallerResponse) hijackRender(ctx context.Context, w h r.payload.Installer.Close() } -func (svc *Service) DownloadSoftwareInstaller(ctx context.Context, id uint) (*fleet.DownloadSoftwareInstallerPayload, error) { +func (svc *Service) DownloadSoftwareInstaller(ctx context.Context, titleID uint, teamID *uint) (*fleet.DownloadSoftwareInstallerPayload, error) { // skipauth: No authorization check needed due to implementation returning // only license error. svc.authz.SkipAuthorization(ctx) diff --git a/server/service/software_installers_test.go b/server/service/software_installers_test.go index 991e8e89e6..65329a332a 100644 --- a/server/service/software_installers_test.go +++ b/server/service/software_installers_test.go @@ -10,6 +10,7 @@ import ( "github.com/fleetdm/fleet/v4/server/mock" "github.com/fleetdm/fleet/v4/server/ptr" "github.com/fleetdm/fleet/v4/server/test" + "github.com/stretchr/testify/require" ) func TestSoftwareInstallersAuth(t *testing.T) { @@ -59,7 +60,7 @@ func TestSoftwareInstallersAuth(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := viewer.NewContext(ctx, viewer.Viewer{User: tt.user}) - ds.GetSoftwareInstallerMetadataFunc = func(ctx context.Context, installerID uint) (*fleet.SoftwareInstaller, error) { + ds.GetSoftwareInstallerMetadataByTeamAndTitleIDFunc = func(ctx context.Context, teamID *uint, titleID uint) (*fleet.SoftwareInstaller, error) { return &fleet.SoftwareInstaller{TeamID: tt.teamID}, nil } @@ -79,11 +80,19 @@ func TestSoftwareInstallersAuth(t *testing.T) { return nil, nil } - _, err := svc.DownloadSoftwareInstaller(ctx, 1) - checkAuthErr(t, tt.shouldFailRead, err) + _, err := svc.DownloadSoftwareInstaller(ctx, 1, tt.teamID) + if tt.teamID == nil { + require.Error(t, err) + } else { + checkAuthErr(t, tt.shouldFailRead, err) + } - err = svc.DeleteSoftwareInstaller(ctx, 1) - checkAuthErr(t, tt.shouldFailWrite, err) + err = svc.DeleteSoftwareInstaller(ctx, 1, tt.teamID) + if tt.teamID == nil { + require.Error(t, err) + } else { + checkAuthErr(t, tt.shouldFailWrite, err) + } // TODO: configure test with mock software installer store and add tests to check upload auth }) diff --git a/server/service/software_titles.go b/server/service/software_titles.go index c9e3189802..b39f8c1767 100644 --- a/server/service/software_titles.go +++ b/server/service/software_titles.go @@ -24,7 +24,7 @@ type listSoftwareTitlesResponse struct { Meta *fleet.PaginationMetadata `json:"meta"` Count int `json:"count"` CountsUpdatedAt *time.Time `json:"counts_updated_at"` - SoftwareTitles []fleet.SoftwareTitle `json:"software_titles,omitempty"` + SoftwareTitles []fleet.SoftwareTitle `json:"software_titles"` Err error `json:"error,omitempty"` } @@ -43,6 +43,9 @@ func listSoftwareTitlesEndpoint(ctx context.Context, request interface{}, svc fl latest = *sw.CountsUpdatedAt } } + if len(titles) == 0 { + titles = []fleet.SoftwareTitle{} + } listResp := listSoftwareTitlesResponse{ SoftwareTitles: titles, Count: count, From 54dbdf322a6870bdc81e0fd2e41107025a9c1a9b Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Fri, 10 May 2024 15:44:49 -0300 Subject: [PATCH 33/56] add version to trigger install endpoint and include script contents (#18915) - add `_version_` instead of hardcoding `v1` for the endpoint to enqueue an install - include scripts content in the response for `/api/latest/fleet/software/titles/:id` as documented # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements) - [x] Added/updated tests - [x] Manual QA for all new/changed functionality --- server/datastore/mysql/software_installers.go | 36 +++++++++------ .../mysql/software_installers_test.go | 45 +++++++++++++++++++ server/fleet/software_installer.go | 4 +- server/service/handler.go | 2 +- server/service/integration_mdm_test.go | 16 +++---- 5 files changed, 78 insertions(+), 25 deletions(-) diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 8f13b5547a..9bef5471bb 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -184,22 +184,30 @@ WHERE func (ds *Datastore) GetSoftwareInstallerMetadataByTeamAndTitleID(ctx context.Context, teamID *uint, titleID uint) (*fleet.SoftwareInstaller, error) { query := ` SELECT - si.id, - si.team_id, - si.title_id, - si.storage_id, - si.filename, - si.version, - si.install_script_content_id, - si.pre_install_query, - si.post_install_script_content_id, - si.uploaded_at, - COALESCE(st.name, '') AS software_title + si.id, + si.team_id, + si.title_id, + si.storage_id, + si.filename, + si.version, + si.install_script_content_id, + si.pre_install_query, + si.post_install_script_content_id, + si.uploaded_at, + inst.contents AS install_script, + COALESCE(pisnt.contents, '') AS post_install_script, + COALESCE(st.name, '') AS software_title FROM - software_installers si - LEFT OUTER JOIN software_titles st ON st.id = si.title_id + software_installers si + LEFT OUTER JOIN software_titles st ON st.id = si.title_id + LEFT OUTER JOIN + script_contents inst + ON inst.id = si.install_script_content_id + LEFT OUTER JOIN + script_contents pisnt + ON pisnt.id = si.post_install_script_content_id WHERE - title_id = ? AND global_or_team_id = ?` + si.title_id = ? AND si.global_or_team_id = ?` var tmID uint if teamID != nil { diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go index ac0f844b7d..62cd225a84 100644 --- a/server/datastore/mysql/software_installers_test.go +++ b/server/datastore/mysql/software_installers_test.go @@ -30,6 +30,7 @@ func TestSoftwareInstallers(t *testing.T) { {"SoftwareInstallerDetails", testListSoftwareInstallerDetails}, {"GetSoftwareInstallResults", testGetSoftwareInstallResult}, {"CleanupUnusedSoftwareInstallers", testCleanupUnusedSoftwareInstallers}, + {"GetSoftwareInstallerMetadataByTeamAndTitleID", testGetSoftwareInstallerMetadataByTeamAndTitleID}, } for _, c := range cases { @@ -413,3 +414,47 @@ func testCleanupUnusedSoftwareInstallers(t *testing.T, ds *Datastore) { require.NoError(t, err) assertExisting(nil) } + +func testGetSoftwareInstallerMetadataByTeamAndTitleID(t *testing.T, ds *Datastore) { + ctx := context.Background() + team, err := ds.NewTeam(ctx, &fleet.Team{Name: "team 2"}) + require.NoError(t, err) + + installerID, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + Title: "foo", + Source: "bar", + InstallScript: "echo install", + PostInstallScript: "echo post-install", + PreInstallQuery: "SELECT 1", + TeamID: &team.ID, + Filename: "foo.pkg", + }) + require.NoError(t, err) + installerMeta, err := ds.GetSoftwareInstallerMetadata(ctx, installerID) + require.NoError(t, err) + + metaByTeamAndTitle, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, &team.ID, *installerMeta.TitleID) + require.NoError(t, err) + require.Equal(t, "echo install", metaByTeamAndTitle.InstallScript) + require.Equal(t, "echo post-install", metaByTeamAndTitle.PostInstallScript) + require.EqualValues(t, installerID, metaByTeamAndTitle.InstallerID) + require.Equal(t, "SELECT 1", metaByTeamAndTitle.PreInstallQuery) + + installerID, err = ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + Title: "bar", + Source: "bar", + InstallScript: "echo install", + TeamID: &team.ID, + Filename: "foo.pkg", + }) + require.NoError(t, err) + installerMeta, err = ds.GetSoftwareInstallerMetadata(ctx, installerID) + require.NoError(t, err) + + metaByTeamAndTitle, err = ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, &team.ID, *installerMeta.TitleID) + require.NoError(t, err) + require.Equal(t, "echo install", metaByTeamAndTitle.InstallScript) + require.Equal(t, "", metaByTeamAndTitle.PostInstallScript) + require.EqualValues(t, installerID, metaByTeamAndTitle.InstallerID) + require.Equal(t, "", metaByTeamAndTitle.PreInstallQuery) +} diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index 5bf411dafa..e91909ae4b 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -78,13 +78,13 @@ type SoftwareInstaller struct { // InstallerID is the unique identifier for the software package metadata in Fleet. InstallerID uint `json:"installer_id" db:"id"` // InstallScript is the script to run to install the software package. - InstallScript string `json:"install_script" db:"-"` + InstallScript string `json:"install_script" db:"install_script"` // InstallScriptContentID is the ID of the install script content. InstallScriptContentID uint `json:"-" db:"install_script_content_id"` // PreInstallQuery is the query to run as a condition to installing the software package. PreInstallQuery string `json:"pre_install_query" db:"pre_install_query"` // PostInstallScript is the script to run after installing the software package. - PostInstallScript string `json:"post_install_script" db:"-"` + PostInstallScript string `json:"post_install_script" db:"post_install_script"` // PostInstallScriptContentID is the ID of the post-install script content. PostInstallScriptContentID *uint `json:"-" db:"post_install_script_content_id"` // StorageID is the unique identifier for the software package in the software installer store. diff --git a/server/service/handler.go b/server/service/handler.go index e358041844..4d0bae48ab 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -370,7 +370,7 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC ue.GET("/api/_version_/fleet/software/titles", listSoftwareTitlesEndpoint, listSoftwareTitlesRequest{}) ue.GET("/api/_version_/fleet/software/titles/{id:[0-9]+}", getSoftwareTitleEndpoint, getSoftwareTitleRequest{}) - ue.POST("/api/v1/fleet/hosts/{host_id:[0-9]+}/software/install/{software_title_id:[0-9]+}", installSoftwareTitleEndpoint, installSoftwareRequest{}) + ue.POST("/api/_version_/fleet/hosts/{host_id:[0-9]+}/software/install/{software_title_id:[0-9]+}", installSoftwareTitleEndpoint, installSoftwareRequest{}) // Sofware installers ue.GET("/api/_version_/fleet/software/{title_id:[0-9]+}/package", getSoftwareInstallerEndpoint, getSoftwareInstallerRequest{}) diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 8e415a10ff..87b644a6b3 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -8722,7 +8722,7 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerNewInstallRequestPlatform } var resp installSoftwareResponse - s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/hosts/%d/software/install/%d", host.ID, softwareTitles[kind]), nil, wantStatus, &resp) + s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/install/%d", host.ID, softwareTitles[kind]), nil, wantStatus, &resp) } } } @@ -8733,7 +8733,7 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerHostRequests() { var resp installSoftwareResponse // non-existent host - s.DoJSON("POST", "/api/v1/fleet/hosts/1/software/install/1", nil, http.StatusNotFound, &resp) + s.DoJSON("POST", "/api/latest/fleet/hosts/1/software/install/1", nil, http.StatusNotFound, &resp) // create a host that doesn't have fleetd installed h, err := s.ds.NewHost(context.Background(), &fleet.Host{ @@ -8750,14 +8750,14 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerHostRequests() { // request fails resp = installSoftwareResponse{} - s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/hosts/%d/software/install/1", h.ID), nil, http.StatusUnprocessableEntity, &resp) + s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/install/1", h.ID), nil, http.StatusUnprocessableEntity, &resp) // host installs fleetd setOrbitEnrollment(t, h, s.ds) // request fails because of non-existent title resp = installSoftwareResponse{} - s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/hosts/%d/software/install/1", h.ID), nil, http.StatusBadRequest, &resp) + s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/install/1", h.ID), nil, http.StatusBadRequest, &resp) payload := &fleet.UploadSoftwareInstallerPayload{ InstallScript: "another install script", @@ -8771,17 +8771,17 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerHostRequests() { // install script request succeeds titleID := getSoftwareTitleID(t, s.ds, payload.Title, "deb_packages") resp = installSoftwareResponse{} - s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/hosts/%d/software/install/%d", h.ID, titleID), nil, http.StatusAccepted, &resp) + s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/install/%d", h.ID, titleID), nil, http.StatusAccepted, &resp) // Get the results, should be pending getHostSoftwareResp := getHostSoftwareResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/hosts/%d/software", h.ID), nil, http.StatusOK, &getHostSoftwareResp) + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", h.ID), nil, http.StatusOK, &getHostSoftwareResp) require.Len(t, getHostSoftwareResp.Software, 1) require.NotNil(t, getHostSoftwareResp.Software[0].LastInstall) installUUID := getHostSoftwareResp.Software[0].LastInstall.InstallUUID gsirr := getSoftwareInstallResultsResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/software/install/results/%s", installUUID), nil, http.StatusOK, &gsirr) + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/software/install/results/%s", installUUID), nil, http.StatusOK, &gsirr) require.NoError(t, gsirr.Err) require.NotNil(t, gsirr.Results) results := gsirr.Results @@ -8790,7 +8790,7 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerHostRequests() { // status is reflected in software title response titleResp := getSoftwareTitleResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/software/titles/%d", titleID), nil, http.StatusOK, &titleResp) + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/software/titles/%d", titleID), nil, http.StatusOK, &titleResp) // TODO: confirm expected behavior of the title response host counts (unspecified) require.Zero(t, titleResp.SoftwareTitle.HostsCount) require.Nil(t, titleResp.SoftwareTitle.CountsUpdatedAt) From 413be14d26c8c283248f65cc236a6c5b7bbed4be Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Mon, 13 May 2024 08:25:25 -0300 Subject: [PATCH 34/56] minor fixes for software installers (#18927) - pass the right query param in the UI for the filters - pass the team id to the filter hosts endpoint --- .../SoftwarePackageCard.tsx | 2 +- server/service/hosts.go | 2 +- server/service/integration_mdm_test.go | 43 +++++++++++++------ 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx index 8491b0fb39..ac9061e743 100644 --- a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx +++ b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx @@ -63,7 +63,7 @@ const PackageStatusCount = ({ const displayData = STATUS_DISPLAY_OPTIONS[status]; const linkUrl = `${PATHS.MANAGE_HOSTS}?${buildQueryStringFromParams({ software_title_id: softwareId, - software_title_status: status, + software_status: status, team_id: teamId, })}`; return ( diff --git a/server/service/hosts.go b/server/service/hosts.go index 92a8e35f9c..e81f211d18 100644 --- a/server/service/hosts.go +++ b/server/service/hosts.go @@ -106,7 +106,7 @@ func listHostsEndpoint(ctx context.Context, request interface{}, svc fleet.Servi if req.Opts.SoftwareTitleIDFilter != nil { var err error - softwareTitle, err = svc.SoftwareTitleByID(ctx, *req.Opts.SoftwareTitleIDFilter, nil) + softwareTitle, err = svc.SoftwareTitleByID(ctx, *req.Opts.SoftwareTitleIDFilter, req.Opts.TeamFilter) if err != nil { return listHostsResponse{Err: err}, nil } diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 87b644a6b3..b90426af2e 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -8731,6 +8731,13 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerNewInstallRequestPlatform func (s *integrationMDMTestSuite) TestSoftwareInstallerHostRequests() { t := s.T() + var createTeamResp teamResponse + s.DoJSON("POST", "/api/latest/fleet/teams", &fleet.Team{ + Name: t.Name(), + }, http.StatusOK, &createTeamResp) + require.NotZero(t, createTeamResp.Team.ID) + teamID := &createTeamResp.Team.ID + var resp installSoftwareResponse // non-existent host s.DoJSON("POST", "/api/latest/fleet/hosts/1/software/install/1", nil, http.StatusNotFound, &resp) @@ -8747,6 +8754,8 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerHostRequests() { Platform: "linux", }) require.NoError(t, err) + err = s.ds.AddHostsToTeam(context.Background(), teamID, []uint{h.ID}) + require.NoError(t, err) // request fails resp = installSoftwareResponse{} @@ -8754,6 +8763,11 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerHostRequests() { // host installs fleetd setOrbitEnrollment(t, h, s.ds) + // TODO(roberto) setOrbitEnrollment is a helper function that silently + // sets the team_id to NULL. We need to refactor it to accept a + // parameter with an optional team value. + err = s.ds.AddHostsToTeam(context.Background(), teamID, []uint{h.ID}) + require.NoError(t, err) // request fails because of non-existent title resp = installSoftwareResponse{} @@ -8765,6 +8779,7 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerHostRequests() { PostInstallScript: "another post install script", Filename: "ruby.deb", Title: "ruby", + TeamID: teamID, } s.uploadSoftwareInstaller(payload, http.StatusOK, "") @@ -8790,7 +8805,7 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerHostRequests() { // status is reflected in software title response titleResp := getSoftwareTitleResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/software/titles/%d", titleID), nil, http.StatusOK, &titleResp) + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/software/titles/%d", titleID), nil, http.StatusOK, &titleResp, "team_id", strconv.Itoa(int(*teamID))) // TODO: confirm expected behavior of the title response host counts (unspecified) require.Zero(t, titleResp.SoftwareTitle.HostsCount) require.Nil(t, titleResp.SoftwareTitle.CountsUpdatedAt) @@ -8805,28 +8820,28 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerHostRequests() { // status is reflected in list hosts responses and counts when filtering by software title and status var listResp listHostsResponse - s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "pending", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID))) + s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) require.Len(t, listResp.Hosts, 1) require.Equal(t, h.ID, listResp.Hosts[0].ID) var countResp countHostsResponse - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "pending", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID))) + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) require.Equal(t, 1, countResp.Count) listResp = listHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "failed", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID))) + s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "failed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) require.Len(t, listResp.Hosts, 0) countResp = countHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "failed", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID))) + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "failed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) require.Equal(t, 0, countResp.Count) listResp = listHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "installed", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID))) + s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "installed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) require.Len(t, listResp.Hosts, 0) countResp = countHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "installed", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID))) + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "installed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) require.Equal(t, 0, countResp.Count) var labelResp createLabelResponse @@ -8842,32 +8857,32 @@ func (s *integrationMDMTestSuite) TestSoftwareInstallerHostRequests() { require.Equal(t, h.ID, listResp.Hosts[0].ID) countResp = countHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "pending", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) require.Equal(t, 1, countResp.Count) listResp = listHostsResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", "pending", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID))) + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) require.Len(t, listResp.Hosts, 1) require.Equal(t, h.ID, listResp.Hosts[0].ID) countResp = countHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "pending", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) require.Equal(t, 1, countResp.Count) listResp = listHostsResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", "installed", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID))) + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", "installed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) require.Len(t, listResp.Hosts, 0) countResp = countHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "installed", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "installed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) require.Equal(t, 0, countResp.Count) listResp = listHostsResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", "failed", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID))) + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", "failed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) require.Len(t, listResp.Hosts, 0) countResp = countHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "failed", "team_id", "0", "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "failed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) require.Equal(t, 0, countResp.Count) // filter validations From 5dafea01c8d6530ded50bea8eecf3e7eda070878 Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Mon, 13 May 2024 09:36:13 -0500 Subject: [PATCH 35/56] Fix unreleased UI bugs for software installers (#18928) --- .../SoftwareInstallDetails.tsx | 1 + .../components/VersionCell/VersionCell.tsx | 2 +- .../HostDetailsPage/HostDetailsPage.tsx | 4 +- .../hosts/details/cards/Software/Software.tsx | 2 +- .../SoftwareDetailsModal.tsx | 50 +++++++++++-------- .../SoftwareDetailsModal/_styles.scss | 11 ++++ 6 files changed, 45 insertions(+), 25 deletions(-) diff --git a/frontend/pages/SoftwarePage/components/SoftwareInstallDetails/SoftwareInstallDetails.tsx b/frontend/pages/SoftwarePage/components/SoftwareInstallDetails/SoftwareInstallDetails.tsx index 4c9c5fa77b..ca735dcf5a 100644 --- a/frontend/pages/SoftwarePage/components/SoftwareInstallDetails/SoftwareInstallDetails.tsx +++ b/frontend/pages/SoftwarePage/components/SoftwareInstallDetails/SoftwareInstallDetails.tsx @@ -86,6 +86,7 @@ export const SoftwareInstallDetails = ({ }, { refetchOnWindowFocus: false, + staleTime: 3000, select: (data) => data.results, } ); diff --git a/frontend/pages/SoftwarePage/components/VersionCell/VersionCell.tsx b/frontend/pages/SoftwarePage/components/VersionCell/VersionCell.tsx index a21e2dc7c7..45441e779f 100644 --- a/frontend/pages/SoftwarePage/components/VersionCell/VersionCell.tsx +++ b/frontend/pages/SoftwarePage/components/VersionCell/VersionCell.tsx @@ -48,7 +48,7 @@ const VersionCell = ({ // only one version, no need for tooltip const cellText = generateText(versions); - if (!versions) { + if (!versions || versions.length <= 1) { return <>{cellText}; } diff --git a/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx b/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx index 001b59ffca..5e7e16ee43 100644 --- a/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx +++ b/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx @@ -853,9 +853,7 @@ const HostDetailsPage = ({ router={router} queryParams={queryParams} pathname={location.pathname} - onShowSoftwareDetails={(software) => - setSelectedSoftwareDetails(software) - } + onShowSoftwareDetails={setSelectedSoftwareDetails} /> {host?.platform === "darwin" && macadmins?.munki?.version && ( ); }; -export default SoftwareCard; +export default React.memo(SoftwareCard); diff --git a/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/SoftwareDetailsModal.tsx b/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/SoftwareDetailsModal.tsx index 3633901f13..ba20637ce4 100644 --- a/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/SoftwareDetailsModal.tsx +++ b/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/SoftwareDetailsModal.tsx @@ -13,6 +13,8 @@ import Button from "components/buttons/Button"; import DataSet from "components/DataSet"; import { dateAgo } from "utilities/date_format"; +import { SoftwareInstallDetails } from "pages/SoftwarePage/components/SoftwareInstallDetails"; + const baseClass = "software-details-modal"; const generateVulnerabilitiesValue = (vulnerabilities: string[]) => { @@ -40,7 +42,7 @@ const SoftwareDetailsInfo = ({ source, bundleIdentifier, }: ISoftwareDetailsInfoProps) => { - const { vulnerabilities } = installedVersion; + const { vulnerabilities, installed_paths } = installedVersion; return (
@@ -57,19 +59,21 @@ const SoftwareDetailsInfo = ({ /> )}
-
- - {installedVersion.installed_paths.map((path) => ( - {path} - ))} -
- } - /> -
+ {!!installed_paths?.length && ( +
+ + {installed_paths.map((path) => ( + {path} + ))} +
+ } + /> +
+ )} {vulnerabilities && vulnerabilities.length !== 0 && (
{ + const installUuid = software.last_install?.install_uuid || ""; + const renderSoftwareDetails = () => { const { installed_versions } = software; @@ -98,16 +104,18 @@ const SoftwareDetailsModal = ({ // software type atm. if (!installed_versions || installed_versions.length === 0) { return ( - +
+ +
); } return (
- {installed_versions.map((installedVersion) => { + {installed_versions?.map((installedVersion) => { return ( Install Details {renderSoftwareDetails()} - test 2 + + + ); diff --git a/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/_styles.scss b/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/_styles.scss index 6b241115a0..1b2374dd61 100644 --- a/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/_styles.scss +++ b/frontend/pages/hosts/details/cards/Software/SoftwareDetailsModal/_styles.scss @@ -1,4 +1,7 @@ .software-details-modal { + &__software-details { + margin-top: 24px; + } &__details-info { display: flex; @@ -43,4 +46,12 @@ white-space: nowrap; } } + + .modal__content { + margin-top: $pad-small; + } + + .react-tabs__tab-panel { + margin-top: $pad-large; + } } From 180a32454a8f9f922028520db8c95d3f09d47b71 Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Mon, 13 May 2024 12:36:17 -0500 Subject: [PATCH 36/56] Additional UI integration and fix unreleased bugs in software installers UI: Part 2 (#18943) --- .../DeleteSoftwareModal.tsx | 9 +++-- .../SoftwarePackageCard.tsx | 10 +++++- .../SoftwareTitleDetailsPage.tsx | 17 +++++++++ .../details/DeviceUserPage/DeviceUserPage.tsx | 2 ++ .../HostDetailsPage/HostDetailsPage.tsx | 2 ++ .../HostSoftwareTable/HostSoftwareTable.tsx | 1 + .../Software/HostSoftwareTableConfig.tsx | 4 ++- .../InstallStatusCell/InstallStatusCell.tsx | 6 ++-- .../hosts/details/cards/Software/Software.tsx | 35 +++++++++++++++---- 9 files changed, 72 insertions(+), 14 deletions(-) diff --git a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/DeleteSoftwareModal/DeleteSoftwareModal.tsx b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/DeleteSoftwareModal/DeleteSoftwareModal.tsx index 2ab014cdf6..23d03c5852 100644 --- a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/DeleteSoftwareModal/DeleteSoftwareModal.tsx +++ b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/DeleteSoftwareModal/DeleteSoftwareModal.tsx @@ -1,4 +1,4 @@ -import React, { useContext } from "react"; +import React, { useCallback, useContext } from "react"; import softwareAPI from "services/entities/software"; import { NotificationContext } from "context/notification"; @@ -12,24 +12,27 @@ interface IDeleteSoftwareModalProps { softwareId: number; teamId: number; onExit: () => void; + onSuccess: () => void; } const DeleteSoftwareModal = ({ softwareId, teamId, onExit, + onSuccess, }: IDeleteSoftwareModalProps) => { const { renderFlash } = useContext(NotificationContext); - const onDeleteSoftware = async () => { + const onDeleteSoftware = useCallback(async () => { try { await softwareAPI.deleteSoftwarePackage(softwareId, teamId); renderFlash("success", "Software deleted successfully!"); + onSuccess(); } catch { renderFlash("error", "Couldn't delete. Please try again."); } onExit(); - }; + }, [softwareId, teamId, renderFlash, onSuccess, onExit]); return ( diff --git a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx index ac9061e743..0972192b77 100644 --- a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx +++ b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx @@ -1,4 +1,4 @@ -import React, { useContext, useState } from "react"; +import React, { useCallback, useContext, useState } from "react"; import endpoints from "utilities/endpoints"; import { SoftwareInstallStatus, ISoftwarePackage } from "interfaces/software"; @@ -93,12 +93,14 @@ interface ISoftwarePackageCardProps { softwarePackage: ISoftwarePackage; softwareId: number; teamId: number; + onDelete: () => void; } const SoftwarePackageCard = ({ softwarePackage, softwareId, teamId, + onDelete, }: ISoftwarePackageCardProps) => { const { isGlobalAdmin, @@ -120,6 +122,11 @@ const SoftwarePackageCard = ({ setShowDeleteModal(true); }; + const onSuccess = useCallback(() => { + setShowDeleteModal(false); + onDelete(); + }, [onDelete]); + const showActions = isGlobalAdmin || isGlobalMaintainer || isTeamAdmin || isTeamMaintainer; @@ -203,6 +210,7 @@ const SoftwarePackageCard = ({ softwareId={softwareId} teamId={teamId} onExit={() => setShowDeleteModal(false)} + onSuccess={onSuccess} /> )} diff --git a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwareTitleDetailsPage.tsx b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwareTitleDetailsPage.tsx index f772b14056..772d189ece 100644 --- a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwareTitleDetailsPage.tsx +++ b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwareTitleDetailsPage.tsx @@ -6,6 +6,8 @@ import { useErrorHandler } from "react-error-boundary"; import { RouteComponentProps } from "react-router"; import { AxiosError } from "axios"; +import paths from "router/paths"; + import useTeamIdParam from "hooks/useTeamIdParam"; import { AppContext } from "context/app"; @@ -74,6 +76,7 @@ const SoftwareTitleDetailsPage = ({ data: softwareTitle, isLoading: isSoftwareTitleLoading, isError: isSoftwareTitleError, + refetch: refetchSoftwareTitle, } = useQuery< ISoftwareTitleResponse, AxiosError, @@ -94,6 +97,19 @@ const SoftwareTitleDetailsPage = ({ } ); + const onDeleteInstaller = useCallback(() => { + if (softwareTitle?.versions?.length) { + refetchSoftwareTitle(); + return; + } + // redirect to software titles page if no versions are available + if (teamIdForApi && teamIdForApi > 0) { + router.push(paths.SOFTWARE_TITLES.concat(`?team_id=${teamIdForApi}`)); + } else { + router.push(paths.SOFTWARE_TITLES); + } + }, [refetchSoftwareTitle, router, softwareTitle, teamIdForApi]); + const onTeamChange = useCallback( (teamId: number) => { handleTeamChange(teamId); @@ -156,6 +172,7 @@ const SoftwareTitleDetailsPage = ({ softwarePackage={softwareTitle.software_package} softwareId={softwareId} teamId={currentTeamId} + onDelete={onDeleteInstaller} /> )} {isPremiumTier && ( diff --git a/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx b/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx index 5e7e16ee43..62eb51cde4 100644 --- a/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx +++ b/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx @@ -849,11 +849,13 @@ const HostDetailsPage = ({ {host?.platform === "darwin" && macadmins?.munki?.version && (
); diff --git a/frontend/pages/hosts/details/cards/Software/HostSoftwareTableConfig.tsx b/frontend/pages/hosts/details/cards/Software/HostSoftwareTableConfig.tsx index ba724f6f46..60d27f7f4d 100644 --- a/frontend/pages/hosts/details/cards/Software/HostSoftwareTableConfig.tsx +++ b/frontend/pages/hosts/details/cards/Software/HostSoftwareTableConfig.tsx @@ -79,6 +79,7 @@ interface ISoftwareTableHeadersProps { onSelectAction: (software: IHostSoftware, action: string) => void; canInstall: boolean; router: InjectedRouter; + teamId: number; } // NOTE: cellProps come from react-table @@ -88,6 +89,7 @@ export const generateSoftwareTableHeaders = ({ installingSoftwareId, onSelectAction, canInstall, + teamId, }: ISoftwareTableHeadersProps): ISoftwareTableConfig[] => { const tableHeaders: ISoftwareTableConfig[] = [ { @@ -100,7 +102,7 @@ export const generateSoftwareTableHeaders = ({ const { id, name, source } = cellProps.row.original; const softwareTitleDetailsPath = PATHS.SOFTWARE_TITLE_DETAILS( - id.toString() + id.toString().concat(`?team_id=${teamId}`) ); return ( diff --git a/frontend/pages/hosts/details/cards/Software/InstallStatusCell/InstallStatusCell.tsx b/frontend/pages/hosts/details/cards/Software/InstallStatusCell/InstallStatusCell.tsx index abad97d41c..7ea5fb3495 100644 --- a/frontend/pages/hosts/details/cards/Software/InstallStatusCell/InstallStatusCell.tsx +++ b/frontend/pages/hosts/details/cards/Software/InstallStatusCell/InstallStatusCell.tsx @@ -68,10 +68,10 @@ const InstallStatusCell = ({ }: IInstallStatusCellProps) => { let displayStatus: IStatusValue; - if (packageToInstall) { - displayStatus = "avaiableForInstall"; - } else if (status !== null) { + if (status !== null) { displayStatus = status; + } else if (packageToInstall) { + displayStatus = "avaiableForInstall"; } else { return ; } diff --git a/frontend/pages/hosts/details/cards/Software/Software.tsx b/frontend/pages/hosts/details/cards/Software/Software.tsx index ca59d2051c..a19c7b3acb 100644 --- a/frontend/pages/hosts/details/cards/Software/Software.tsx +++ b/frontend/pages/hosts/details/cards/Software/Software.tsx @@ -11,6 +11,7 @@ import deviceAPI, { IDeviceSoftwareQueryParams, IGetDeviceSoftwareResponse, } from "services/entities/device_user"; +import { getErrorReason } from "interfaces/errors"; import { IHostSoftware, ISoftware } from "interfaces/software"; import { DEFAULT_USE_QUERY_OPTIONS } from "utilities/constants"; import { NotificationContext } from "context/notification"; @@ -33,6 +34,7 @@ export interface ITableSoftware extends Omit { interface ISoftwareCardProps { /** This is the host id or the device token */ id: number | string; + isFleetdHost: boolean; router: InjectedRouter; queryParams?: { page?: string; @@ -41,22 +43,26 @@ interface ISoftwareCardProps { order_direction?: "asc" | "desc"; }; pathname: string; + /** Team id for the host */ + teamId: number; onShowSoftwareDetails?: (software: IHostSoftware) => void; isSoftwareEnabled?: boolean; isMyDevicePage?: boolean; } const DEFAULT_SEARCH_QUERY = ""; -const DEFAULT_SORT_DIRECTION = "desc"; +const DEFAULT_SORT_DIRECTION = "asc"; const DEFAULT_SORT_HEADER = "name"; const DEFAULT_PAGE = 0; const DEFAULT_PAGE_SIZE = 20; const SoftwareCard = ({ id, + isFleetdHost, router, queryParams, pathname, + teamId = 0, onShowSoftwareDetails, isSoftwareEnabled = false, isMyDevicePage = false, @@ -86,6 +92,7 @@ const SoftwareCard = ({ isLoading: hostSoftwareLoading, isError: hostSoftwareError, isFetching: hostSoftwareFetching, + refetch: refetchHostSoftware, } = useQuery< IGetHostSoftwareResponse, AxiosError, @@ -116,6 +123,7 @@ const SoftwareCard = ({ isLoading: deviceSoftwareLoading, isError: deviceSoftwareError, isFetching: deviceSoftwareFetching, + refetch: refetchDeviceSoftware, } = useQuery< IGetDeviceSoftwareResponse, AxiosError, @@ -139,10 +147,17 @@ const SoftwareCard = ({ } ); - const canInstallSoftware = Boolean( - isGlobalAdmin || isGlobalMaintainer || isTeamAdmin || isTeamMaintainer + const refetchSoftware = useMemo( + () => (isMyDevicePage ? refetchDeviceSoftware : refetchHostSoftware), + [isMyDevicePage, refetchDeviceSoftware, refetchHostSoftware] ); + const canInstallSoftware = + isFleetdHost && + Boolean( + isGlobalAdmin || isGlobalMaintainer || isTeamAdmin || isTeamMaintainer + ); + const installHostSoftwarePackage = useCallback( async (softwareId: number) => { setInstallingSoftwareId(softwareId); @@ -152,12 +167,18 @@ const SoftwareCard = ({ "success", "Software is installing or will install when the host comes online." ); - } catch { - renderFlash("error", "Couldn't install. Please try again."); + } catch (e) { + const reason = getErrorReason(e); + if (reason.includes("fleetd installed")) { + renderFlash("error", reason); + } else { + renderFlash("error", "Couldn't install. Please try again."); + } } setInstallingSoftwareId(null); + refetchSoftware(); }, - [id, renderFlash] + [id, renderFlash, refetchSoftware] ); const onSelectAction = useCallback( @@ -184,6 +205,7 @@ const SoftwareCard = ({ installingSoftwareId, canInstall: canInstallSoftware, onSelectAction, + teamId, }); }, [ isMyDevicePage, @@ -191,6 +213,7 @@ const SoftwareCard = ({ installingSoftwareId, canInstallSoftware, onSelectAction, + teamId, ]); const renderSoftwareTable = () => { From 0debd186734b45cf30f16cd69118550a9a22dfd8 Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Mon, 13 May 2024 17:32:59 -0500 Subject: [PATCH 37/56] Fix unreleased issues in software installers UI: Part 3 (#18972) --- .../SoftwarePackageCard.tsx | 62 ++++++++++++++----- .../AddSoftwareForm/AddSoftwareForm.tsx | 1 - .../SoftwarePage/components/icons/index.ts | 2 + .../hosts/details/cards/Software/_styles.scss | 10 +-- frontend/services/entities/software.ts | 21 +++++++ frontend/services/index.ts | 6 +- 6 files changed, 81 insertions(+), 21 deletions(-) diff --git a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx index 0972192b77..cea4bfc05d 100644 --- a/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx +++ b/frontend/pages/SoftwarePage/SoftwareTitleDetailsPage/SoftwarePackageCard/SoftwarePackageCard.tsx @@ -1,9 +1,16 @@ import React, { useCallback, useContext, useState } from "react"; -import endpoints from "utilities/endpoints"; -import { SoftwareInstallStatus, ISoftwarePackage } from "interfaces/software"; +import FileSaver from "file-saver"; + import PATHS from "router/paths"; + import { AppContext } from "context/app"; +import { NotificationContext } from "context/notification"; + +import { SoftwareInstallStatus, ISoftwarePackage } from "interfaces/software"; + +import softwareAPI from "services/entities/software"; + import { buildQueryStringFromParams } from "utilities/url"; import { internationalTimeFormat } from "utilities/helpers"; import { uploadedFromNow } from "utilities/date_format"; @@ -109,6 +116,8 @@ const SoftwarePackageCard = ({ isTeamMaintainer, } = useContext(AppContext); + const { renderFlash } = useContext(NotificationContext); + const [showAdvancedOptionsModal, setShowAdvancedOptionsModal] = useState( false ); @@ -122,18 +131,45 @@ const SoftwarePackageCard = ({ setShowDeleteModal(true); }; - const onSuccess = useCallback(() => { + const onDeleteSuccess = useCallback(() => { setShowDeleteModal(false); onDelete(); }, [onDelete]); + const onDownloadClick = useCallback(async () => { + try { + const resp = await softwareAPI.downloadSoftwarePackage( + softwareId, + teamId + ); + const contentLength = parseInt(resp.headers["content-length"], 10); + if (contentLength !== resp.data.size) { + throw new Error( + `Byte size (${resp.data.size}) does not match content-length header (${contentLength})` + ); + } + const filename = softwarePackage.name; + const file = new File([resp.data], filename, { + type: "application/octet-stream", + }); + if (file.size === 0) { + throw new Error("Downloaded file is empty"); + } + if (file.size !== resp.data.size) { + throw new Error( + `File size (${file.size}) does not match expected size (${resp.data.size})` + ); + } + FileSaver.saveAs(file); + } catch (e) { + console.log(e); + renderFlash("error", "Couldn’t download. Please try again."); + } + }, [renderFlash, softwareId, softwarePackage.name, teamId]); + const showActions = isGlobalAdmin || isGlobalMaintainer || isTeamAdmin || isTeamMaintainer; - const downloadUrl = `/api${endpoints.SOFTWARE_PACKAGE( - softwareId - )}?${buildQueryStringFromParams({ alt: "media", team_id: teamId })}`; - return (
@@ -185,13 +221,9 @@ const SoftwarePackageCard = ({ {/* TODO: make a component for download icons */} - - - + @@ -210,7 +242,7 @@ const SoftwarePackageCard = ({ softwareId={softwareId} teamId={teamId} onExit={() => setShowDeleteModal(false)} - onSuccess={onSuccess} + onSuccess={onDeleteSuccess} /> )} diff --git a/frontend/pages/SoftwarePage/components/AddSoftwareForm/AddSoftwareForm.tsx b/frontend/pages/SoftwarePage/components/AddSoftwareForm/AddSoftwareForm.tsx index 05ece55866..4f7fee9dbc 100644 --- a/frontend/pages/SoftwarePage/components/AddSoftwareForm/AddSoftwareForm.tsx +++ b/frontend/pages/SoftwarePage/components/AddSoftwareForm/AddSoftwareForm.tsx @@ -70,7 +70,6 @@ const AddSoftwareForm = ({ onCancel, onSubmit, }: IAddSoftwareFormProps) => { - console.log("rerender"); const [showPreInstallCondition, setShowPreInstallCondition] = useState(false); const [showPostInstallScript, setShowPostInstallScript] = useState(false); const [formData, setFormData] = useState({ diff --git a/frontend/pages/SoftwarePage/components/icons/index.ts b/frontend/pages/SoftwarePage/components/icons/index.ts index f101759110..8ac710679f 100644 --- a/frontend/pages/SoftwarePage/components/icons/index.ts +++ b/frontend/pages/SoftwarePage/components/icons/index.ts @@ -17,6 +17,7 @@ import Word from "./Word"; import Zoom from "./Zoom"; import ChromeOS from "./ChromeOS"; import LinuxOS from "./LinuxOS"; +// import Falcon from "./Falcon"; // TODO: Add Falcon icon svg // SOFTWARE_NAME_TO_ICON_MAP list "special" applications that have a defined // icon for them, keys refer to application names, and are intended to be fuzzy @@ -33,6 +34,7 @@ export const SOFTWARE_NAME_TO_ICON_MAP = { "visual studio code": VisualStudioCode, "microsoft word": Word, zoom: Zoom, + // falcon: Falcon, darwin: MacOS, windows: WindowsOS, chrome: ChromeOS, diff --git a/frontend/pages/hosts/details/cards/Software/_styles.scss b/frontend/pages/hosts/details/cards/Software/_styles.scss index f74c1e4eac..780245ac89 100644 --- a/frontend/pages/hosts/details/cards/Software/_styles.scss +++ b/frontend/pages/hosts/details/cards/Software/_styles.scss @@ -1,10 +1,12 @@ .software-card { - .table-container__search-input { width: 325px; // Custom to fit placeholder text } - .data-table-block .data-table__wrapper { - overflow-x: scroll; - } + // // TODO: Addingoverflow-x: scroll to the table clips the actions dropdown at the bottom of the + // // table. Find a solution that allows dropdown menu to be displayed over the bottom of the table + // // in the y-axis when the table is scrollable in the x-axis. + // .data-table-block .data-table__wrapper { + // overflow-x: scroll; + // } } diff --git a/frontend/services/entities/software.ts b/frontend/services/entities/software.ts index b81539a288..40be25ddc5 100644 --- a/frontend/services/entities/software.ts +++ b/frontend/services/entities/software.ts @@ -1,3 +1,5 @@ +import { AxiosResponse } from "axios"; + import { snakeCase, reduce } from "lodash"; import sendRequest from "services"; @@ -211,6 +213,25 @@ export default { return sendRequest("DELETE", path); }, + downloadSoftwarePackage: ( + softwareTitleId: number, + teamId: number + ): Promise => { + const path = `${endpoints.SOFTWARE_PACKAGE( + softwareTitleId + )}?${buildQueryStringFromParams({ alt: "media", team_id: teamId })}`; + + return sendRequest( + "GET", + path, + undefined, + "blob", + undefined, + undefined, + true // return raw response + ); + }, + getSoftwareInstallResult: (installUuid: string) => { const { SOFTWARE_INSTALL_RESULTS } = endpoints; const path = SOFTWARE_INSTALL_RESULTS(installUuid); diff --git a/frontend/services/index.ts b/frontend/services/index.ts index 7152f042f2..acca7f824f 100644 --- a/frontend/services/index.ts +++ b/frontend/services/index.ts @@ -8,7 +8,8 @@ export const sendRequest = async ( data?: unknown, responseType: AxiosResponseType = "json", timeout?: number, - skipParseError?: boolean + skipParseError?: boolean, + returnRaw?: boolean ) => { const { origin } = global.window.location; @@ -27,6 +28,9 @@ export const sendRequest = async ( }, }); + if (returnRaw) { + return response; + } return response.data; } catch (error) { if (skipParseError) { From 3579e5a250aa504526c987fc495bbdd651e1fa70 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Tue, 14 May 2024 08:37:07 -0400 Subject: [PATCH 38/56] Software installers: backend cleanup tasks part 1 (#18955) --- server/datastore/mysql/activities.go | 2 +- server/datastore/mysql/software.go | 33 +- server/datastore/mysql/software_installers.go | 93 +-- server/service/integration_enterprise_test.go | 708 +++++++++++++++++- server/service/integration_mdm_test.go | 617 --------------- server/service/testing_client.go | 13 +- 6 files changed, 759 insertions(+), 707 deletions(-) diff --git a/server/datastore/mysql/activities.go b/server/datastore/mysql/activities.go index 8005f995f0..a37ece00d8 100644 --- a/server/datastore/mysql/activities.go +++ b/server/datastore/mysql/activities.go @@ -306,7 +306,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint hsi.host_id = :host_id AND hsi.pre_install_query_output IS NULL AND hsi.install_script_exit_code IS NULL - `, softwareInstallerHostStatusNamedQuery("")), + `, softwareInstallerHostStatusNamedQuery("hsi", "")), } seconds := int(scripts.MaxServerWaitTime.Seconds()) diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index d2fbb3e670..4e276f2c50 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -1721,33 +1721,38 @@ func (ds *Datastore) ListCVEs(ctx context.Context, maxAge time.Duration) ([]flee return result, nil } +// tblAlias is the table alias to use as prefix for the host_script_installs +// column names, no prefix used if empty. // colAlias is the name to be assigned to the computed status column, pass // empty to have the value only, no column alias set. -func softwareInstallerHostStatusNamedQuery(colAlias string) string { +func softwareInstallerHostStatusNamedQuery(tblAlias, colAlias string) string { + if tblAlias != "" { + tblAlias += "." + } if colAlias != "" { colAlias = " AS " + colAlias } return fmt.Sprintf(` CASE - WHEN hsi.post_install_script_exit_code IS NOT NULL AND - hsi.post_install_script_exit_code = 0 THEN :software_status_installed + WHEN %[1]spost_install_script_exit_code IS NOT NULL AND + %[1]spost_install_script_exit_code = 0 THEN :software_status_installed - WHEN hsi.post_install_script_exit_code IS NOT NULL AND - hsi.post_install_script_exit_code != 0 THEN :software_status_failed + WHEN %[1]spost_install_script_exit_code IS NOT NULL AND + %[1]spost_install_script_exit_code != 0 THEN :software_status_failed - WHEN hsi.install_script_exit_code IS NOT NULL AND - hsi.install_script_exit_code = 0 THEN :software_status_installed + WHEN %[1]sinstall_script_exit_code IS NOT NULL AND + %[1]sinstall_script_exit_code = 0 THEN :software_status_installed - WHEN hsi.install_script_exit_code IS NOT NULL AND - hsi.install_script_exit_code != 0 THEN :software_status_failed + WHEN %[1]sinstall_script_exit_code IS NOT NULL AND + %[1]sinstall_script_exit_code != 0 THEN :software_status_failed - WHEN hsi.pre_install_query_output IS NOT NULL AND - hsi.pre_install_query_output = '' THEN :software_status_failed + WHEN %[1]spre_install_query_output IS NOT NULL AND + %[1]spre_install_query_output = '' THEN :software_status_failed - WHEN hsi.host_id IS NOT NULL THEN :software_status_pending + WHEN %[1]shost_id IS NOT NULL THEN :software_status_pending ELSE NULL -- not installed from Fleet installer - END %s `, colAlias) + END %[2]s `, tblAlias, colAlias) } func (ds *Datastore) ListHostSoftware(ctx context.Context, hostID uint, includeAvailableForInstall bool, opts fleet.ListOptions) ([]*fleet.HostSoftwareWithInstaller, *fleet.PaginationMetadata, error) { @@ -1792,7 +1797,7 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, hostID uint, includeA ) OR -- or software install has been attempted on host hsi.host_id IS NOT NULL ) -`, softwareInstallerHostStatusNamedQuery("status")) +`, softwareInstallerHostStatusNamedQuery("hsi", "status")) const stmtAvailable = ` SELECT diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 9bef5471bb..8895411e9a 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -197,7 +197,7 @@ SELECT inst.contents AS install_script, COALESCE(pisnt.contents, '') AS post_install_script, COALESCE(st.name, '') AS software_title -FROM +FROM software_installers si LEFT OUTER JOIN software_titles st ON st.id = si.title_id LEFT OUTER JOIN @@ -206,7 +206,7 @@ FROM LEFT OUTER JOIN script_contents pisnt ON pisnt.id = si.post_install_script_content_id -WHERE +WHERE si.title_id = ? AND si.global_or_team_id = ?` var tmID uint @@ -314,7 +314,7 @@ func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID ui } func (ds *Datastore) GetSoftwareInstallResults(ctx context.Context, resultsUUID string) (*fleet.HostSoftwareInstallerResult, error) { - query := ` + query := fmt.Sprintf(` SELECT hsi.execution_id AS execution_id, COALESCE(hsi.pre_install_query_output, '') AS pre_install_query_output, @@ -324,20 +324,7 @@ SELECT h.computer_name AS host_display_name, st.name AS software_title, st.id AS software_title_id, - COALESCE(CASE - WHEN hsi.post_install_script_exit_code IS NOT NULL AND - hsi.post_install_script_exit_code = 0 THEN ? -- installed - WHEN hsi.post_install_script_exit_code IS NOT NULL AND - hsi.post_install_script_exit_code != 0 THEN ? -- failed - WHEN hsi.install_script_exit_code IS NOT NULL AND - hsi.install_script_exit_code = 0 THEN ? -- installed - WHEN hsi.install_script_exit_code IS NOT NULL AND - hsi.install_script_exit_code != 0 THEN ? -- failed - WHEN hsi.pre_install_query_output IS NOT NULL AND - hsi.pre_install_query_output = '' THEN ? -- failed - WHEN hsi.host_id IS NOT NULL THEN ? -- pending - ELSE NULL -- not installed from Fleet installer - END, '') AS status, + COALESCE(%s, '') AS status, si.filename AS software_package, h.team_id AS host_team_id, hsi.user_id AS user_id @@ -347,11 +334,21 @@ FROM JOIN software_installers si ON si.id = hsi.software_installer_id JOIN software_titles st ON si.title_id = st.id WHERE - hsi.execution_id = ? - ` + hsi.execution_id = :execution_id + `, softwareInstallerHostStatusNamedQuery("hsi", "")) + + stmt, args, err := sqlx.Named(query, map[string]any{ + "execution_id": resultsUUID, + "software_status_failed": fleet.SoftwareInstallerFailed, + "software_status_pending": fleet.SoftwareInstallerPending, + "software_status_installed": fleet.SoftwareInstallerInstalled, + }) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "build named query for get software install results") + } var dest fleet.HostSoftwareInstallerResult - err := sqlx.GetContext(ctx, ds.reader(ctx), &dest, query, fleet.SoftwareInstallerInstalled, fleet.SoftwareInstallerFailed, fleet.SoftwareInstallerInstalled, fleet.SoftwareInstallerFailed, fleet.SoftwareInstallerFailed, fleet.SoftwareInstallerPending, resultsUUID) + err = sqlx.GetContext(ctx, ds.reader(ctx), &dest, stmt, args...) if err != nil { if err == sql.ErrNoRows { return nil, ctxerr.Wrap(ctx, notFound("HostSoftwareInstallerResult"), "get host software installer results") @@ -362,42 +359,18 @@ WHERE return &dest, nil } -func tmplNamedSQLCaseHostSoftwareInstallStatus(alias string) string { - return fmt.Sprintf(` - CASE WHEN %[1]s.post_install_script_exit_code IS NOT NULL - AND %[1]s.post_install_script_exit_code = 0 THEN - :installed - WHEN %[1]s.post_install_script_exit_code IS NOT NULL - AND %[1]s.post_install_script_exit_code != 0 THEN - :failed - WHEN %[1]s.install_script_exit_code IS NOT NULL - AND %[1]s.install_script_exit_code = 0 THEN - :installed - WHEN %[1]s.install_script_exit_code IS NOT NULL - AND %[1]s.install_script_exit_code != 0 THEN - :failed - WHEN %[1]s.pre_install_query_output IS NOT NULL - AND %[1]s.pre_install_query_output = '' THEN - :failed - WHEN %[1]s.host_id IS NOT NULL THEN - :pending - ELSE - NULL -- not installed from Fleet installer - END`, alias) -} - func (ds *Datastore) GetSummaryHostSoftwareInstalls(ctx context.Context, installerID uint) (*fleet.SoftwareInstallerStatusSummary, error) { var dest fleet.SoftwareInstallerStatusSummary stmt := fmt.Sprintf(` SELECT - COALESCE(SUM( IF(status = :pending, 1, 0)), 0) AS pending, - COALESCE(SUM( IF(status = :failed, 1, 0)), 0) AS failed, - COALESCE(SUM( IF(status = :installed, 1, 0)), 0) AS installed + COALESCE(SUM( IF(status = :software_status_pending, 1, 0)), 0) AS pending, + COALESCE(SUM( IF(status = :software_status_failed, 1, 0)), 0) AS failed, + COALESCE(SUM( IF(status = :software_status_installed, 1, 0)), 0) AS installed FROM ( SELECT software_installer_id, - %s AS status + %s FROM host_software_installs hsi WHERE @@ -406,16 +379,16 @@ WHERE SELECT max(id) -- ensure we use only the most recently created install attempt for each host FROM host_software_installs - WHERE + WHERE software_installer_id = :installer_id GROUP BY - host_id)) s`, tmplNamedSQLCaseHostSoftwareInstallStatus("hsi")) + host_id)) s`, softwareInstallerHostStatusNamedQuery("hsi", "status")) query, args, err := sqlx.Named(stmt, map[string]interface{}{ - "installer_id": installerID, - "pending": fleet.SoftwareInstallerPending, - "failed": fleet.SoftwareInstallerFailed, - "installed": fleet.SoftwareInstallerInstalled, + "installer_id": installerID, + "software_status_pending": fleet.SoftwareInstallerPending, + "software_status_failed": fleet.SoftwareInstallerFailed, + "software_status_installed": fleet.SoftwareInstallerInstalled, }) if err != nil { return nil, ctxerr.Wrap(ctx, err, "get summary host software installs: named query") @@ -446,14 +419,14 @@ WHERE GROUP BY host_id, software_installer_id) AND (%s) = :status) hss ON hss.host_id = h.id -`, tmplNamedSQLCaseHostSoftwareInstallStatus("hsi")) +`, softwareInstallerHostStatusNamedQuery("hsi", "")) return sqlx.Named(stmt, map[string]interface{}{ - "status": status, - "installer_id": installerID, - "installed": fleet.SoftwareInstallerInstalled, - "failed": fleet.SoftwareInstallerFailed, - "pending": fleet.SoftwareInstallerPending, + "status": status, + "installer_id": installerID, + "software_status_installed": fleet.SoftwareInstallerInstalled, + "software_status_failed": fleet.SoftwareInstallerFailed, + "software_status_pending": fleet.SoftwareInstallerPending, }) } diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 9e8201b099..0e5e8fd974 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -5,13 +5,16 @@ import ( "context" "database/sql" "encoding/base64" + "encoding/hex" "encoding/json" "errors" "fmt" "io" + "mime/multipart" "net/http" "net/http/httptest" "os" + "path/filepath" "reflect" "sort" "strconv" @@ -8679,14 +8682,20 @@ func (s *integrationEnterpriseTestSuite) TestListHostSoftware() { ctx := context.Background() t := s.T() - // clean up any software titles from previous tests - mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { - _, err := q.ExecContext(ctx, `DELETE FROM software_titles`) - return err - }) - token := "good_token" - host := createHostAndDeviceToken(t, s.ds, token) + host := createOrbitEnrolledHost(t, "linux", "host1", s.ds) + createDeviceTokenForHost(t, s.ds, host.ID, token) + + // no software yet + var getHostSw getHostSoftwareResponse + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", host.ID), nil, http.StatusOK, &getHostSw) + require.Len(t, getHostSw.Software, 0) + + var getDeviceSw getDeviceSoftwareResponse + res := s.DoRawNoAuth("GET", "/api/latest/fleet/device/"+token+"/software", nil, http.StatusOK) + err := json.NewDecoder(res.Body).Decode(&getDeviceSw) + require.NoError(t, err) + require.Len(t, getDeviceSw.Software, 0) // create some software for that host software := []fleet.Software{ @@ -8694,20 +8703,20 @@ func (s *integrationEnterpriseTestSuite) TestListHostSoftware() { {Name: "foo", Version: "0.0.2", Source: "chrome_extensions"}, {Name: "bar", Version: "0.0.1", Source: "apps"}, } - _, err := s.ds.UpdateHostSoftware(ctx, host.ID, software) + _, err = s.ds.UpdateHostSoftware(ctx, host.ID, software) require.NoError(t, err) err = s.ds.ReconcileSoftwareTitles(ctx) require.NoError(t, err) - var getHostSw getHostSoftwareResponse + getHostSw = getHostSoftwareResponse{} s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", host.ID), nil, http.StatusOK, &getHostSw) require.Len(t, getHostSw.Software, 2) // foo and bar require.Equal(t, getHostSw.Software[0].Name, "bar") require.Equal(t, getHostSw.Software[1].Name, "foo") require.Len(t, getHostSw.Software[1].InstalledVersions, 2) - var getDeviceSw getDeviceSoftwareResponse - res := s.DoRawNoAuth("GET", "/api/latest/fleet/device/"+token+"/software", nil, http.StatusOK) + res = s.DoRawNoAuth("GET", "/api/latest/fleet/device/"+token+"/software", nil, http.StatusOK) + getDeviceSw = getDeviceSoftwareResponse{} err = json.NewDecoder(res.Body).Decode(&getDeviceSw) require.NoError(t, err) require.Len(t, getDeviceSw.Software, 2) // foo and bar @@ -8715,19 +8724,694 @@ func (s *integrationEnterpriseTestSuite) TestListHostSoftware() { require.Equal(t, getDeviceSw.Software[1].Name, "foo") require.Len(t, getDeviceSw.Software[1].InstalledVersions, 2) + // create a software installer, not installed on the host + payload := &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "install", + Filename: "ruby.deb", + } + s.uploadSoftwareInstaller(payload, http.StatusOK, "") + titleID := getSoftwareTitleID(t, s.ds, "ruby", "deb_packages") + + // available installer is returned by user-authenticated endpoint + getHostSw = getHostSoftwareResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", host.ID), nil, http.StatusOK, &getHostSw) + require.Len(t, getHostSw.Software, 3) // foo, bar and ruby.deb + require.Equal(t, getHostSw.Software[0].Name, "bar") + require.Equal(t, getHostSw.Software[1].Name, "foo") + require.Equal(t, getHostSw.Software[2].Name, "ruby") + require.Len(t, getHostSw.Software[1].InstalledVersions, 2) + require.NotNil(t, getHostSw.Software[2].PackageAvailableForInstall) + require.Equal(t, "ruby.deb", *getHostSw.Software[2].PackageAvailableForInstall) + require.Nil(t, getHostSw.Software[2].Status) + + // available installer is not returned by device-authenticated endpoint + res = s.DoRawNoAuth("GET", "/api/latest/fleet/device/"+token+"/software", nil, http.StatusOK) + getDeviceSw = getDeviceSoftwareResponse{} + err = json.NewDecoder(res.Body).Decode(&getDeviceSw) + require.NoError(t, err) + require.Len(t, getDeviceSw.Software, 2) // foo and bar + require.Equal(t, getDeviceSw.Software[0].Name, "bar") + require.Equal(t, getDeviceSw.Software[1].Name, "foo") + require.Len(t, getDeviceSw.Software[1].InstalledVersions, 2) + require.Nil(t, getDeviceSw.Software[0].PackageAvailableForInstall) + require.Nil(t, getDeviceSw.Software[1].PackageAvailableForInstall) + + // request installation on the host + var installResp installSoftwareResponse + s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/install/%d", + host.ID, titleID), nil, http.StatusAccepted, &installResp) + + // still returned by user-authenticated endpoint, now pending + getHostSw = getHostSoftwareResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", host.ID), nil, http.StatusOK, &getHostSw) + require.Len(t, getHostSw.Software, 3) // foo, bar and ruby.deb + require.Equal(t, getHostSw.Software[0].Name, "bar") + require.Equal(t, getHostSw.Software[1].Name, "foo") + require.Equal(t, getHostSw.Software[2].Name, "ruby") + require.Len(t, getHostSw.Software[1].InstalledVersions, 2) + require.NotNil(t, getHostSw.Software[2].PackageAvailableForInstall) + require.Equal(t, "ruby.deb", *getHostSw.Software[2].PackageAvailableForInstall) + require.NotNil(t, getHostSw.Software[2].Status) + require.Equal(t, fleet.SoftwareInstallerPending, *getHostSw.Software[2].Status) + + // now returned by device-authenticated endpoin + res = s.DoRawNoAuth("GET", "/api/latest/fleet/device/"+token+"/software", nil, http.StatusOK) + getDeviceSw = getDeviceSoftwareResponse{} + err = json.NewDecoder(res.Body).Decode(&getDeviceSw) + require.NoError(t, err) + require.Len(t, getDeviceSw.Software, 3) // foo, bar and ruby + require.Equal(t, getDeviceSw.Software[0].Name, "bar") + require.Equal(t, getDeviceSw.Software[1].Name, "foo") + require.Equal(t, getDeviceSw.Software[2].Name, "ruby") + require.Len(t, getDeviceSw.Software[1].InstalledVersions, 2) + require.Nil(t, getDeviceSw.Software[2].PackageAvailableForInstall) + require.NotNil(t, getDeviceSw.Software[2].Status) + require.Equal(t, fleet.SoftwareInstallerPending, *getDeviceSw.Software[2].Status) + // test with a query + getHostSw = getHostSoftwareResponse{} s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", host.ID), nil, http.StatusOK, &getHostSw, "query", "foo") require.Len(t, getHostSw.Software, 1) // foo only require.Equal(t, getHostSw.Software[0].Name, "foo") require.Len(t, getHostSw.Software[0].InstalledVersions, 2) + res = s.DoRawNoAuth("GET", "/api/latest/fleet/device/"+token+"/software?query=bar", nil, http.StatusOK) + getDeviceSw = getDeviceSoftwareResponse{} err = json.NewDecoder(res.Body).Decode(&getDeviceSw) require.NoError(t, err) require.Len(t, getDeviceSw.Software, 1) // bar only require.Equal(t, getDeviceSw.Software[0].Name, "bar") require.Len(t, getDeviceSw.Software[0].InstalledVersions, 1) +} - // TODO(mna): more advanced integration tests with Software Installers once the APIs are in place. +func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerUploadDownloadAndDelete() { + t := s.T() + + openFile := func(name string) *os.File { + f, err := os.Open(filepath.Join("testdata", "software-installers", name)) + require.NoError(t, err) + return f + } + + var expectBytes []byte + var expectLen int + f := openFile("ruby.deb") + st, err := f.Stat() + require.NoError(t, err) + expectLen = int(st.Size()) + require.Equal(t, expectLen, 11340) + expectBytes = make([]byte, expectLen) + n, err := f.Read(expectBytes) + require.NoError(t, err) + require.Equal(t, n, expectLen) + f.Close() + + checkDownloadResponse := func(t *testing.T, r *http.Response, expectedFilename string) { + require.Equal(t, "application/octet-stream", r.Header.Get("Content-Type")) + require.Equal(t, fmt.Sprintf(`attachment;filename="%s"`, expectedFilename), r.Header.Get("Content-Disposition")) + require.NotZero(t, r.ContentLength) + require.Equal(t, expectLen, int(r.ContentLength)) + b, err := io.ReadAll(r.Body) + require.NoError(t, err) + require.Equal(t, expectLen, len(b)) + require.Equal(t, expectBytes, b) + } + + checkSoftwareTitle := func(t *testing.T, title string, source string) uint { + var id uint + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(context.Background(), q, &id, `SELECT id FROM software_titles WHERE name = ? AND source = ? AND browser = ''`, title, source) + }) + return id + } + + checkScriptContentsID := func(t *testing.T, id uint, expectedContents string) { + var contents string + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(context.Background(), q, &contents, `SELECT contents FROM script_contents WHERE id = ?`, id) + }) + require.Equal(t, expectedContents, contents) + } + + checkSoftwareInstaller := func(t *testing.T, payload *fleet.UploadSoftwareInstallerPayload) (installerID uint, titleID uint) { + var id uint + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + var tid uint + if payload.TeamID != nil { + tid = *payload.TeamID + } + return sqlx.GetContext(context.Background(), q, &id, `SELECT id FROM software_installers WHERE global_or_team_id = ? AND filename = ?`, tid, payload.Filename) + }) + require.NotZero(t, id) + + meta, err := s.ds.GetSoftwareInstallerMetadata(context.Background(), id) + require.NoError(t, err) + + if payload.TeamID != nil { + require.Equal(t, *payload.TeamID, *meta.TeamID) + } else { + require.Nil(t, meta.TeamID) + } + + checkScriptContentsID(t, meta.InstallScriptContentID, payload.InstallScript) + + if payload.PostInstallScript != "" { + require.NotNil(t, meta.PostInstallScriptContentID) + checkScriptContentsID(t, *meta.PostInstallScriptContentID, payload.PostInstallScript) + } else { + require.Nil(t, meta.PostInstallScriptContentID) + } + + require.Equal(t, payload.PreInstallQuery, meta.PreInstallQuery) + require.Equal(t, payload.StorageID, meta.StorageID) + require.Equal(t, payload.Filename, meta.Name) + require.Equal(t, payload.Version, meta.Version) + require.Equal(t, checkSoftwareTitle(t, payload.Title, "deb_packages"), *meta.TitleID) + require.NotZero(t, meta.UploadedAt) + + return meta.InstallerID, *meta.TitleID + } + + t.Run("upload no team software installer", func(t *testing.T) { + payload := &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "some install script", + PreInstallQuery: "some pre install query", + PostInstallScript: "some post install script", + Filename: "ruby.deb", + // additional fields below are pre-populated so we can re-use the payload later for the test assertions + Title: "ruby", + Version: "1:2.5.1", + Source: "deb_packages", + StorageID: "df06d9ce9e2090d9cb2e8cd1f4d7754a803dc452bf93e3204e3acd3b95508628", + } + + s.uploadSoftwareInstaller(payload, http.StatusOK, "") + + // check activity + s.lastActivityOfTypeMatches(fleet.ActivityTypeAddedSoftware{}.ActivityName(), `{"software_title": "ruby", "software_package": "ruby.deb", "team_name": null, "team_id": null}`, 0) + + // check the software installer + _, titleID := checkSoftwareInstaller(t, payload) + + // upload again fails + s.uploadSoftwareInstaller(payload, http.StatusConflict, "already exists") + + // download the installer + s.Do("GET", fmt.Sprintf("/api/latest/fleet/software/%d/package?alt=media", titleID), nil, http.StatusBadRequest) + + // delete the installer + s.Do("DELETE", fmt.Sprintf("/api/latest/fleet/software/%d/package", titleID), nil, http.StatusBadRequest) + }) + + t.Run("create team software installer", func(t *testing.T) { + var createTeamResp teamResponse + s.DoJSON("POST", "/api/latest/fleet/teams", &fleet.Team{ + Name: t.Name(), + }, http.StatusOK, &createTeamResp) + require.NotZero(t, createTeamResp.Team.ID) + + payload := &fleet.UploadSoftwareInstallerPayload{ + TeamID: &createTeamResp.Team.ID, + InstallScript: "another install script", + PreInstallQuery: "another pre install query", + PostInstallScript: "another post install script", + Filename: "ruby.deb", + // additional fields below are pre-populated so we can re-use the payload later for the test assertions + Title: "ruby", + Version: "1:2.5.1", + Source: "deb_packages", + StorageID: "df06d9ce9e2090d9cb2e8cd1f4d7754a803dc452bf93e3204e3acd3b95508628", + } + + s.uploadSoftwareInstaller(payload, http.StatusOK, "") + + // check the software installer + installerID, titleID := checkSoftwareInstaller(t, payload) + + // check activity + s.lastActivityOfTypeMatches(fleet.ActivityTypeAddedSoftware{}.ActivityName(), fmt.Sprintf(`{"software_title": "ruby", "software_package": "ruby.deb", "team_name": "%s", "team_id": %d}`, createTeamResp.Team.Name, createTeamResp.Team.ID), 0) + + // upload again fails + s.uploadSoftwareInstaller(payload, http.StatusConflict, "already exists") + + // download the installer + r := s.Do("GET", fmt.Sprintf("/api/latest/fleet/software/%d/package?alt=media", titleID), nil, http.StatusOK, "team_id", fmt.Sprintf("%d", *payload.TeamID)) + checkDownloadResponse(t, r, payload.Filename) + + // create an orbit host, assign to team and request to download the installer + host := createOrbitEnrolledHost(t, "windows", "orbit-host-team", s.ds) + require.NoError(t, s.ds.AddHostsToTeam(context.Background(), &createTeamResp.Team.ID, []uint{host.ID})) + r = s.Do("POST", "/api/fleet/orbit/software_install/package?alt=media", orbitDownloadSoftwareInstallerRequest{ + InstallerID: installerID, + OrbitNodeKey: *host.OrbitNodeKey, + }, http.StatusOK) + checkDownloadResponse(t, r, payload.Filename) + + // delete the installer + s.Do("DELETE", fmt.Sprintf("/api/latest/fleet/software/%d/package", titleID), nil, http.StatusNoContent, "team_id", fmt.Sprintf("%d", *payload.TeamID)) + + // check activity + s.lastActivityOfTypeMatches(fleet.ActivityTypeDeletedSoftware{}.ActivityName(), fmt.Sprintf(`{"software_title": "ruby", "software_package": "ruby.deb", "team_name": "%s", "team_id": %d}`, createTeamResp.Team.Name, createTeamResp.Team.ID), 0) + }) +} + +func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerNewInstallRequestPlatformValidation() { + t := s.T() + + hostsByPlatform := map[string]*fleet.Host{ + "linux": nil, "darwin": nil, "windows": nil, + } + + tm, err := s.ds.NewTeam(context.Background(), &fleet.Team{ + Name: t.Name(), + Description: "desc", + }) + require.NoError(t, err) + + for platform := range hostsByPlatform { + h, err := s.ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now().Add(-1 * time.Minute), + OsqueryHostID: ptr.String(t.Name() + uuid.New().String()), + NodeKey: ptr.String(t.Name() + uuid.New().String()), + Hostname: fmt.Sprintf("%sfoo.local", t.Name()), + Platform: platform, + }) + require.NoError(t, err) + setOrbitEnrollment(t, h, s.ds) + + err = s.ds.AddHostsToTeam(context.Background(), &tm.ID, []uint{h.ID}) + require.NoError(t, err) + + hostsByPlatform[platform] = h + } + + softwareTitles := map[string]uint{ + "deb": 0, "msi": 0, "exe": 0, "pkg": 0, + } + + for kind := range softwareTitles { + // TODO(roberto): we need real binaries for exe, msi and pkg to + // perform the API calls. + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + ctx := context.Background() + installScript := fmt.Sprintf(`echo '%s'`, kind) + res, err := q.ExecContext(ctx, `INSERT INTO script_contents (md5_checksum, contents) VALUES (UNHEX(md5(?)), ?)`, installScript, installScript) + if err != nil { + return err + } + scriptContentID, _ := res.LastInsertId() + + res, err = q.ExecContext(ctx, `INSERT INTO software_titles (name, source) VALUES ('foo', ?)`, kind) + if err != nil { + return err + } + titleID, _ := res.LastInsertId() + softwareTitles[kind] = uint(titleID) + + _, err = q.ExecContext(ctx, ` + INSERT INTO software_installers + (title_id, filename, version, install_script_content_id, storage_id, team_id, global_or_team_id, pre_install_query) + VALUES + (?, ?, ?, ?, unhex(?), ?, ?, ?)`, + titleID, fmt.Sprintf("installer.%s", kind), "v1.0.0", scriptContentID, hex.EncodeToString([]byte("test")), tm.ID, tm.ID, "foo") + return err + }) + } + + testCases := []struct { + platform string + supportedInstallers []string + }{ + {"windows", []string{"exe", "msi"}}, + {"darwin", []string{"pkg"}}, + {"linux", []string{"deb"}}, + } + + for _, tc := range testCases { + for platform, host := range hostsByPlatform { + for _, kind := range tc.supportedInstallers { + wantStatus := http.StatusAccepted + if tc.platform != platform { + wantStatus = http.StatusBadRequest + } + + var resp installSoftwareResponse + s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/install/%d", host.ID, softwareTitles[kind]), nil, wantStatus, &resp) + } + } + } +} + +func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerHostRequests() { + t := s.T() + + var createTeamResp teamResponse + s.DoJSON("POST", "/api/latest/fleet/teams", &fleet.Team{ + Name: t.Name(), + }, http.StatusOK, &createTeamResp) + require.NotZero(t, createTeamResp.Team.ID) + teamID := &createTeamResp.Team.ID + + var resp installSoftwareResponse + // non-existent host + s.DoJSON("POST", "/api/latest/fleet/hosts/1/software/install/1", nil, http.StatusNotFound, &resp) + + // create a host that doesn't have fleetd installed + h, err := s.ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now().Add(-1 * time.Minute), + OsqueryHostID: ptr.String(t.Name() + uuid.New().String()), + NodeKey: ptr.String(t.Name() + uuid.New().String()), + Hostname: fmt.Sprintf("%sfoo.local", t.Name()), + Platform: "linux", + }) + require.NoError(t, err) + err = s.ds.AddHostsToTeam(context.Background(), teamID, []uint{h.ID}) + require.NoError(t, err) + + // request fails + resp = installSoftwareResponse{} + s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/install/1", h.ID), nil, http.StatusUnprocessableEntity, &resp) + + // host installs fleetd + setOrbitEnrollment(t, h, s.ds) + // TODO(roberto) setOrbitEnrollment is a helper function that silently + // sets the team_id to NULL. We need to refactor it to accept a + // parameter with an optional team value. + err = s.ds.AddHostsToTeam(context.Background(), teamID, []uint{h.ID}) + require.NoError(t, err) + + // request fails because of non-existent title + resp = installSoftwareResponse{} + s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/install/1", h.ID), nil, http.StatusBadRequest, &resp) + + payload := &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "another install script", + PreInstallQuery: "another pre install query", + PostInstallScript: "another post install script", + Filename: "ruby.deb", + Title: "ruby", + TeamID: teamID, + } + s.uploadSoftwareInstaller(payload, http.StatusOK, "") + + // install script request succeeds + titleID := getSoftwareTitleID(t, s.ds, payload.Title, "deb_packages") + resp = installSoftwareResponse{} + s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/install/%d", h.ID, titleID), nil, http.StatusAccepted, &resp) + + // Get the results, should be pending + getHostSoftwareResp := getHostSoftwareResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", h.ID), nil, http.StatusOK, &getHostSoftwareResp) + require.Len(t, getHostSoftwareResp.Software, 1) + require.NotNil(t, getHostSoftwareResp.Software[0].LastInstall) + installUUID := getHostSoftwareResp.Software[0].LastInstall.InstallUUID + + gsirr := getSoftwareInstallResultsResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/software/install/results/%s", installUUID), nil, http.StatusOK, &gsirr) + require.NoError(t, gsirr.Err) + require.NotNil(t, gsirr.Results) + results := gsirr.Results + require.Equal(t, installUUID, results.InstallUUID) + require.Equal(t, fleet.SoftwareInstallerPending, results.Status) + + // status is reflected in software title response + titleResp := getSoftwareTitleResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/software/titles/%d", titleID), nil, http.StatusOK, &titleResp, "team_id", strconv.Itoa(int(*teamID))) + // TODO: confirm expected behavior of the title response host counts (unspecified) + require.Zero(t, titleResp.SoftwareTitle.HostsCount) + require.Nil(t, titleResp.SoftwareTitle.CountsUpdatedAt) + require.NotNil(t, titleResp.SoftwareTitle.SoftwarePackage) + require.Equal(t, "ruby.deb", titleResp.SoftwareTitle.SoftwarePackage.Name) + require.NotNil(t, titleResp.SoftwareTitle.SoftwarePackage.Status) + require.Equal(t, fleet.SoftwareInstallerStatusSummary{ + Installed: 0, + Pending: 1, + Failed: 0, + }, *titleResp.SoftwareTitle.SoftwarePackage.Status) + + // status is reflected in list hosts responses and counts when filtering by software title and status + var listResp listHostsResponse + s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) + require.Len(t, listResp.Hosts, 1) + require.Equal(t, h.ID, listResp.Hosts[0].ID) + + var countResp countHostsResponse + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) + require.Equal(t, 1, countResp.Count) + + listResp = listHostsResponse{} + s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "failed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) + require.Len(t, listResp.Hosts, 0) + + countResp = countHostsResponse{} + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "failed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) + require.Equal(t, 0, countResp.Count) + + listResp = listHostsResponse{} + s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "installed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) + require.Len(t, listResp.Hosts, 0) + + countResp = countHostsResponse{} + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "installed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) + require.Equal(t, 0, countResp.Count) + + var labelResp createLabelResponse + s.DoJSON("POST", "/api/latest/fleet/labels", &createLabelRequest{fleet.LabelPayload{ + Name: "test", + Hosts: []string{h.Hostname}, + }}, http.StatusOK, &labelResp) + require.NotZero(t, labelResp.Label.ID) + + listResp = listHostsResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp) + require.Len(t, listResp.Hosts, 1) + require.Equal(t, h.ID, listResp.Hosts[0].ID) + + countResp = countHostsResponse{} + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) + require.Equal(t, 1, countResp.Count) + + listResp = listHostsResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) + require.Len(t, listResp.Hosts, 1) + require.Equal(t, h.ID, listResp.Hosts[0].ID) + + countResp = countHostsResponse{} + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) + require.Equal(t, 1, countResp.Count) + + listResp = listHostsResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", "installed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) + require.Len(t, listResp.Hosts, 0) + + countResp = countHostsResponse{} + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "installed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) + require.Equal(t, 0, countResp.Count) + + listResp = listHostsResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", "failed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) + require.Len(t, listResp.Hosts, 0) + + countResp = countHostsResponse{} + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "failed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) + require.Equal(t, 0, countResp.Count) + + // filter validations + r := s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "uninstalled") + require.Contains(t, extractServerErrorText(r.Body), "Invalid software_status") + r = s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "installed") + require.Contains(t, extractServerErrorText(r.Body), "Missing software_title_id") + r = s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "installed", "software_title_id", "1") + require.Contains(t, extractServerErrorText(r.Body), "Missing team_id") + r = s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "installed", "team_id", "1") + require.Contains(t, extractServerErrorText(r.Body), "Missing software_title_id") + r = s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "installed", "team_id", "1", "software_title_id", "1", "software_version_id", "1") + require.Contains(t, extractServerErrorText(r.Body), "Invalid parameters. The combination of software_version_id and software_title_id is not allowed.") + r = s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "installed", "team_id", "1", "software_title_id", "1", "software_id", "1") + require.Contains(t, extractServerErrorText(r.Body), "Invalid parameters. The combination of software_id and software_title_id is not allowed.") + + // TODO(roberto): once we have endpoints to retrieve installers, + // request them using the orbit node key + + // TODO(sarah): test other statuses once we have endpoints to set results via orbit +} + +func (s *integrationEnterpriseTestSuite) TestHostSoftwareInstallResult() { + ctx := context.Background() + t := s.T() + + host := createOrbitEnrolledHost(t, "linux", "", s.ds) + + // create a software installer and some host install requests + payload := &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "install script", + PreInstallQuery: "pre install query", + PostInstallScript: "post install script", + Filename: "ruby.deb", + Title: "ruby", + } + s.uploadSoftwareInstaller(payload, http.StatusOK, "") + titleID := getSoftwareTitleID(t, s.ds, payload.Title, "deb_packages") + + latestInstallUUID := func() string { + var id string + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(ctx, q, &id, `SELECT execution_id FROM host_software_installs ORDER BY id DESC LIMIT 1`) + }) + return id + } + + // create some install requests for the host + installUUIDs := make([]string, 3) + for i := 0; i < len(installUUIDs); i++ { + resp := installSoftwareResponse{} + s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/hosts/%d/software/install/%d", host.ID, titleID), nil, http.StatusAccepted, &resp) + installUUIDs[i] = latestInstallUUID() + } + + type result struct { + HostID uint + InstallUUID string + Status fleet.SoftwareInstallerStatus + } + checkResults := func(want result) { + var resp getSoftwareInstallResultsResponse + s.DoJSON("GET", "/api/v1/fleet/software/install/results/"+want.InstallUUID, nil, http.StatusOK, &resp) + + assert.Equal(t, want.HostID, resp.Results.HostID) + assert.Equal(t, want.InstallUUID, resp.Results.InstallUUID) + assert.Equal(t, want.Status, resp.Results.Status) + } + + s.Do("POST", "/api/fleet/orbit/software_install/result", + json.RawMessage(fmt.Sprintf(`{ + "orbit_node_key": %q, + "install_uuid": %q, + "pre_install_condition_output": "1", + "install_script_exit_code": 1, + "install_script_output": "failed" + }`, *host.OrbitNodeKey, installUUIDs[0])), + http.StatusNoContent) + checkResults(result{ + HostID: host.ID, + InstallUUID: installUUIDs[0], + Status: fleet.SoftwareInstallerFailed, + }) + wantAct := fleet.ActivityTypeInstalledSoftware{ + HostID: host.ID, + HostDisplayName: host.DisplayName(), + SoftwareTitle: payload.Title, + InstallUUID: installUUIDs[0], + Status: string(fleet.SoftwareInstallerFailed), + } + s.lastActivityMatches(wantAct.ActivityName(), string(jsonMustMarshal(t, wantAct)), 0) + + s.Do("POST", "/api/fleet/orbit/software_install/result", + json.RawMessage(fmt.Sprintf(`{ + "orbit_node_key": %q, + "install_uuid": %q, + "pre_install_condition_output": "" + }`, *host.OrbitNodeKey, installUUIDs[1])), + http.StatusNoContent) + checkResults(result{ + HostID: host.ID, + InstallUUID: installUUIDs[1], + Status: fleet.SoftwareInstallerFailed, + }) + wantAct.InstallUUID = installUUIDs[1] + s.lastActivityOfTypeMatches(wantAct.ActivityName(), string(jsonMustMarshal(t, wantAct)), 0) + + s.Do("POST", "/api/fleet/orbit/software_install/result", + json.RawMessage(fmt.Sprintf(`{ + "orbit_node_key": %q, + "install_uuid": %q, + "pre_install_condition_output": "1", + "install_script_exit_code": 0, + "install_script_output": "success", + "post_install_script_exit_code": 0, + "post_install_script_output": "ok" + }`, *host.OrbitNodeKey, installUUIDs[2])), + http.StatusNoContent) + checkResults(result{ + HostID: host.ID, + InstallUUID: installUUIDs[2], + Status: fleet.SoftwareInstallerInstalled, + }) + wantAct.InstallUUID = installUUIDs[2] + wantAct.Status = string(fleet.SoftwareInstallerInstalled) + lastActID := s.lastActivityOfTypeMatches(wantAct.ActivityName(), string(jsonMustMarshal(t, wantAct)), 0) + + // non-existing installation uuid + s.Do("POST", "/api/fleet/orbit/software_install/result", + json.RawMessage(fmt.Sprintf(`{ + "orbit_node_key": %q, + "install_uuid": "uuid-no-such", + "pre_install_condition_output": "" + }`, *host.OrbitNodeKey)), + http.StatusNotFound) + // no new activity created + s.lastActivityOfTypeMatches(wantAct.ActivityName(), string(jsonMustMarshal(t, wantAct)), lastActID) +} + +func (s *integrationEnterpriseTestSuite) uploadSoftwareInstaller(payload *fleet.UploadSoftwareInstallerPayload, expectedStatus int, expectedError string) { + t := s.T() + openFile := func(name string) *os.File { + f, err := os.Open(filepath.Join("testdata", "software-installers", name)) + require.NoError(t, err) + return f + } + + f := openFile(payload.Filename) + defer f.Close() + + payload.InstallerFile = f + + var b bytes.Buffer + w := multipart.NewWriter(&b) + + // add the software field + fw, err := w.CreateFormFile("software", payload.Filename) + require.NoError(t, err) + n, err := io.Copy(fw, payload.InstallerFile) + require.NoError(t, err) + require.NotZero(t, n) + + // add the team_id field + if payload.TeamID != nil { + require.NoError(t, w.WriteField("team_id", fmt.Sprintf("%d", *payload.TeamID))) + } + // add the remaining fields + require.NoError(t, w.WriteField("install_script", payload.InstallScript)) + require.NoError(t, w.WriteField("pre_install_query", payload.PreInstallQuery)) + require.NoError(t, w.WriteField("post_install_script", payload.PostInstallScript)) + + w.Close() + + headers := map[string]string{ + "Content-Type": w.FormDataContentType(), + "Accept": "application/json", + "Authorization": fmt.Sprintf("Bearer %s", s.token), + } + + r := s.DoRawWithHeaders("POST", "/api/latest/fleet/software/package", b.Bytes(), expectedStatus, headers) + if expectedError != "" { + errMsg := extractServerErrorText(r.Body) + require.Contains(t, errMsg, expectedError) + } +} + +func getSoftwareTitleID(t *testing.T, ds *mysql.Datastore, title, source string) uint { + var id uint + mysql.ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(context.Background(), q, &id, `SELECT id FROM software_titles WHERE name = ? AND source = ? AND browser = ''`, title, source) + }) + return id } func genDistributedReqWithPolicyResults(host *fleet.Host, policyResults map[uint]*bool) submitDistributedQueryResultsRequestShim { diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index b90426af2e..7dc1ee4fce 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -6,7 +6,6 @@ import ( "crypto/x509" "database/sql" "encoding/base64" - "encoding/hex" "encoding/json" "encoding/xml" "errors" @@ -363,12 +362,6 @@ func (s *integrationMDMTestSuite) TearDownTest() { _, err := tx.ExecContext(ctx, "DELETE FROM host_mdm_apple_declarations") return err }) - - // clear any lingering software installers - mysql.ExecAdhocSQL(t, s.ds, func(tx sqlx.ExtContext) error { - _, err := tx.ExecContext(ctx, "DELETE FROM software_installers") - return err - }) } func (s *integrationMDMTestSuite) mockDEPResponse(handler http.Handler) { @@ -8466,613 +8459,3 @@ func (s *integrationMDMTestSuite) TestIsServerBitlockerStatus() { require.NotNil(t, hr.Host.MDM.OSSettings.DiskEncryption.Status) require.Equal(t, fleet.DiskEncryptionEnforcing, *hr.Host.MDM.OSSettings.DiskEncryption.Status) } - -func (s *integrationMDMTestSuite) TestSoftwareInstallerUploadDownloadAndDelete() { - t := s.T() - - openFile := func(name string) *os.File { - f, err := os.Open(filepath.Join("testdata", "software-installers", name)) - require.NoError(t, err) - return f - } - - var expectBytes []byte - var expectLen int - f := openFile("ruby.deb") - st, err := f.Stat() - require.NoError(t, err) - expectLen = int(st.Size()) - require.Equal(t, expectLen, 11340) - expectBytes = make([]byte, expectLen) - n, err := f.Read(expectBytes) - require.NoError(t, err) - require.Equal(t, n, expectLen) - f.Close() - - checkDownloadResponse := func(t *testing.T, r *http.Response, expectedFilename string) { - require.Equal(t, "application/octet-stream", r.Header.Get("Content-Type")) - require.Equal(t, fmt.Sprintf(`attachment;filename="%s"`, expectedFilename), r.Header.Get("Content-Disposition")) - require.NotZero(t, r.ContentLength) - require.Equal(t, expectLen, int(r.ContentLength)) - b, err := io.ReadAll(r.Body) - require.NoError(t, err) - require.Equal(t, expectLen, len(b)) - require.Equal(t, expectBytes, b) - } - - checkSoftwareTitle := func(t *testing.T, title string, source string) uint { - var id uint - mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { - return sqlx.GetContext(context.Background(), q, &id, `SELECT id FROM software_titles WHERE name = ? AND source = ? AND browser = ''`, title, source) - }) - return id - } - - checkScriptContentsID := func(t *testing.T, id uint, expectedContents string) { - var contents string - mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { - return sqlx.GetContext(context.Background(), q, &contents, `SELECT contents FROM script_contents WHERE id = ?`, id) - }) - require.Equal(t, expectedContents, contents) - } - - checkSoftwareInstaller := func(t *testing.T, payload *fleet.UploadSoftwareInstallerPayload) (installerID uint, titleID uint) { - var id uint - mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { - var tid uint - if payload.TeamID != nil { - tid = *payload.TeamID - } - return sqlx.GetContext(context.Background(), q, &id, `SELECT id FROM software_installers WHERE global_or_team_id = ? AND filename = ?`, tid, payload.Filename) - }) - require.NotZero(t, id) - - meta, err := s.ds.GetSoftwareInstallerMetadata(context.Background(), id) - require.NoError(t, err) - - if payload.TeamID != nil { - require.Equal(t, *payload.TeamID, *meta.TeamID) - } else { - require.Nil(t, meta.TeamID) - } - - checkScriptContentsID(t, meta.InstallScriptContentID, payload.InstallScript) - - if payload.PostInstallScript != "" { - require.NotNil(t, meta.PostInstallScriptContentID) - checkScriptContentsID(t, *meta.PostInstallScriptContentID, payload.PostInstallScript) - } else { - require.Nil(t, meta.PostInstallScriptContentID) - } - - require.Equal(t, payload.PreInstallQuery, meta.PreInstallQuery) - require.Equal(t, payload.StorageID, meta.StorageID) - require.Equal(t, payload.Filename, meta.Name) - require.Equal(t, payload.Version, meta.Version) - require.Equal(t, checkSoftwareTitle(t, payload.Title, "deb_packages"), *meta.TitleID) - require.NotZero(t, meta.UploadedAt) - - return meta.InstallerID, *meta.TitleID - } - - t.Run("upload no team software installer", func(t *testing.T) { - payload := &fleet.UploadSoftwareInstallerPayload{ - InstallScript: "some install script", - PreInstallQuery: "some pre install query", - PostInstallScript: "some post install script", - Filename: "ruby.deb", - // additional fields below are pre-populated so we can re-use the payload later for the test assertions - Title: "ruby", - Version: "1:2.5.1", - Source: "deb_packages", - StorageID: "df06d9ce9e2090d9cb2e8cd1f4d7754a803dc452bf93e3204e3acd3b95508628", - } - - s.uploadSoftwareInstaller(payload, http.StatusOK, "") - - // check activity - s.lastActivityOfTypeMatches(fleet.ActivityTypeAddedSoftware{}.ActivityName(), `{"software_title": "ruby", "software_package": "ruby.deb", "team_name": null, "team_id": null}`, 0) - - // check the software installer - _, titleID := checkSoftwareInstaller(t, payload) - - // upload again fails - s.uploadSoftwareInstaller(payload, http.StatusConflict, "already exists") - - // download the installer - s.Do("GET", fmt.Sprintf("/api/latest/fleet/software/%d/package?alt=media", titleID), nil, http.StatusBadRequest) - - // delete the installer - s.Do("DELETE", fmt.Sprintf("/api/latest/fleet/software/%d/package", titleID), nil, http.StatusBadRequest) - }) - - t.Run("create team software installer", func(t *testing.T) { - var createTeamResp teamResponse - s.DoJSON("POST", "/api/latest/fleet/teams", &fleet.Team{ - Name: t.Name(), - }, http.StatusOK, &createTeamResp) - require.NotZero(t, createTeamResp.Team.ID) - - payload := &fleet.UploadSoftwareInstallerPayload{ - TeamID: &createTeamResp.Team.ID, - InstallScript: "another install script", - PreInstallQuery: "another pre install query", - PostInstallScript: "another post install script", - Filename: "ruby.deb", - // additional fields below are pre-populated so we can re-use the payload later for the test assertions - Title: "ruby", - Version: "1:2.5.1", - Source: "deb_packages", - StorageID: "df06d9ce9e2090d9cb2e8cd1f4d7754a803dc452bf93e3204e3acd3b95508628", - } - - s.uploadSoftwareInstaller(payload, http.StatusOK, "") - - // check the software installer - installerID, titleID := checkSoftwareInstaller(t, payload) - - // check activity - s.lastActivityOfTypeMatches(fleet.ActivityTypeAddedSoftware{}.ActivityName(), fmt.Sprintf(`{"software_title": "ruby", "software_package": "ruby.deb", "team_name": "%s", "team_id": %d}`, createTeamResp.Team.Name, createTeamResp.Team.ID), 0) - - // upload again fails - s.uploadSoftwareInstaller(payload, http.StatusConflict, "already exists") - - // download the installer - r := s.Do("GET", fmt.Sprintf("/api/latest/fleet/software/%d/package?alt=media", titleID), nil, http.StatusOK, "team_id", fmt.Sprintf("%d", *payload.TeamID)) - checkDownloadResponse(t, r, payload.Filename) - - // create an orbit host, assign to team and request to download the installer - host := createOrbitEnrolledHost(t, "windows", "orbit-host-team", s.ds) - require.NoError(t, s.ds.AddHostsToTeam(context.Background(), &createTeamResp.Team.ID, []uint{host.ID})) - r = s.Do("POST", "/api/fleet/orbit/software_install/package?alt=media", orbitDownloadSoftwareInstallerRequest{ - InstallerID: installerID, - OrbitNodeKey: *host.OrbitNodeKey, - }, http.StatusOK) - checkDownloadResponse(t, r, payload.Filename) - - // delete the installer - s.Do("DELETE", fmt.Sprintf("/api/latest/fleet/software/%d/package", titleID), nil, http.StatusNoContent, "team_id", fmt.Sprintf("%d", *payload.TeamID)) - - // check activity - s.lastActivityOfTypeMatches(fleet.ActivityTypeDeletedSoftware{}.ActivityName(), fmt.Sprintf(`{"software_title": "ruby", "software_package": "ruby.deb", "team_name": "%s", "team_id": %d}`, createTeamResp.Team.Name, createTeamResp.Team.ID), 0) - }) -} - -func (s *integrationMDMTestSuite) TestSoftwareInstallerNewInstallRequestPlatformValidation() { - t := s.T() - - hostsByPlatform := map[string]*fleet.Host{ - "linux": nil, "darwin": nil, "windows": nil, - } - - tm, err := s.ds.NewTeam(context.Background(), &fleet.Team{ - Name: t.Name(), - Description: "desc", - }) - require.NoError(t, err) - - for platform := range hostsByPlatform { - h, err := s.ds.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), - SeenTime: time.Now().Add(-1 * time.Minute), - OsqueryHostID: ptr.String(t.Name() + uuid.New().String()), - NodeKey: ptr.String(t.Name() + uuid.New().String()), - Hostname: fmt.Sprintf("%sfoo.local", t.Name()), - Platform: platform, - }) - require.NoError(t, err) - setOrbitEnrollment(t, h, s.ds) - - err = s.ds.AddHostsToTeam(context.Background(), &tm.ID, []uint{h.ID}) - require.NoError(t, err) - - hostsByPlatform[platform] = h - } - - softwareTitles := map[string]uint{ - "deb": 0, "msi": 0, "exe": 0, "pkg": 0, - } - - for kind := range softwareTitles { - // TODO(roberto): we need real binaries for exe, msi and pkg to - // perform the API calls. - mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { - ctx := context.Background() - installScript := fmt.Sprintf(`echo '%s'`, kind) - res, err := q.ExecContext(ctx, `INSERT INTO script_contents (md5_checksum, contents) VALUES (UNHEX(md5(?)), ?)`, installScript, installScript) - if err != nil { - return err - } - scriptContentID, _ := res.LastInsertId() - - res, err = q.ExecContext(ctx, `INSERT INTO software_titles (name, source) VALUES ('foo', ?)`, kind) - if err != nil { - return err - } - titleID, _ := res.LastInsertId() - softwareTitles[kind] = uint(titleID) - - _, err = q.ExecContext(ctx, ` - INSERT INTO software_installers - (title_id, filename, version, install_script_content_id, storage_id, team_id, global_or_team_id, pre_install_query) - VALUES - (?, ?, ?, ?, unhex(?), ?, ?, ?)`, - titleID, fmt.Sprintf("installer.%s", kind), "v1.0.0", scriptContentID, hex.EncodeToString([]byte("test")), tm.ID, tm.ID, "foo") - return err - }) - } - - testCases := []struct { - platform string - supportedInstallers []string - }{ - {"windows", []string{"exe", "msi"}}, - {"darwin", []string{"pkg"}}, - {"linux", []string{"deb"}}, - } - - for _, tc := range testCases { - for platform, host := range hostsByPlatform { - for _, kind := range tc.supportedInstallers { - wantStatus := http.StatusAccepted - if tc.platform != platform { - wantStatus = http.StatusBadRequest - } - - var resp installSoftwareResponse - s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/install/%d", host.ID, softwareTitles[kind]), nil, wantStatus, &resp) - } - } - } -} - -func (s *integrationMDMTestSuite) TestSoftwareInstallerHostRequests() { - t := s.T() - - var createTeamResp teamResponse - s.DoJSON("POST", "/api/latest/fleet/teams", &fleet.Team{ - Name: t.Name(), - }, http.StatusOK, &createTeamResp) - require.NotZero(t, createTeamResp.Team.ID) - teamID := &createTeamResp.Team.ID - - var resp installSoftwareResponse - // non-existent host - s.DoJSON("POST", "/api/latest/fleet/hosts/1/software/install/1", nil, http.StatusNotFound, &resp) - - // create a host that doesn't have fleetd installed - h, err := s.ds.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - PolicyUpdatedAt: time.Now(), - SeenTime: time.Now().Add(-1 * time.Minute), - OsqueryHostID: ptr.String(t.Name() + uuid.New().String()), - NodeKey: ptr.String(t.Name() + uuid.New().String()), - Hostname: fmt.Sprintf("%sfoo.local", t.Name()), - Platform: "linux", - }) - require.NoError(t, err) - err = s.ds.AddHostsToTeam(context.Background(), teamID, []uint{h.ID}) - require.NoError(t, err) - - // request fails - resp = installSoftwareResponse{} - s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/install/1", h.ID), nil, http.StatusUnprocessableEntity, &resp) - - // host installs fleetd - setOrbitEnrollment(t, h, s.ds) - // TODO(roberto) setOrbitEnrollment is a helper function that silently - // sets the team_id to NULL. We need to refactor it to accept a - // parameter with an optional team value. - err = s.ds.AddHostsToTeam(context.Background(), teamID, []uint{h.ID}) - require.NoError(t, err) - - // request fails because of non-existent title - resp = installSoftwareResponse{} - s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/install/1", h.ID), nil, http.StatusBadRequest, &resp) - - payload := &fleet.UploadSoftwareInstallerPayload{ - InstallScript: "another install script", - PreInstallQuery: "another pre install query", - PostInstallScript: "another post install script", - Filename: "ruby.deb", - Title: "ruby", - TeamID: teamID, - } - s.uploadSoftwareInstaller(payload, http.StatusOK, "") - - // install script request succeeds - titleID := getSoftwareTitleID(t, s.ds, payload.Title, "deb_packages") - resp = installSoftwareResponse{} - s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/install/%d", h.ID, titleID), nil, http.StatusAccepted, &resp) - - // Get the results, should be pending - getHostSoftwareResp := getHostSoftwareResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", h.ID), nil, http.StatusOK, &getHostSoftwareResp) - require.Len(t, getHostSoftwareResp.Software, 1) - require.NotNil(t, getHostSoftwareResp.Software[0].LastInstall) - installUUID := getHostSoftwareResp.Software[0].LastInstall.InstallUUID - - gsirr := getSoftwareInstallResultsResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/software/install/results/%s", installUUID), nil, http.StatusOK, &gsirr) - require.NoError(t, gsirr.Err) - require.NotNil(t, gsirr.Results) - results := gsirr.Results - require.Equal(t, installUUID, results.InstallUUID) - require.Equal(t, fleet.SoftwareInstallerPending, results.Status) - - // status is reflected in software title response - titleResp := getSoftwareTitleResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/software/titles/%d", titleID), nil, http.StatusOK, &titleResp, "team_id", strconv.Itoa(int(*teamID))) - // TODO: confirm expected behavior of the title response host counts (unspecified) - require.Zero(t, titleResp.SoftwareTitle.HostsCount) - require.Nil(t, titleResp.SoftwareTitle.CountsUpdatedAt) - require.NotNil(t, titleResp.SoftwareTitle.SoftwarePackage) - require.Equal(t, "ruby.deb", titleResp.SoftwareTitle.SoftwarePackage.Name) - require.NotNil(t, titleResp.SoftwareTitle.SoftwarePackage.Status) - require.Equal(t, fleet.SoftwareInstallerStatusSummary{ - Installed: 0, - Pending: 1, - Failed: 0, - }, *titleResp.SoftwareTitle.SoftwarePackage.Status) - - // status is reflected in list hosts responses and counts when filtering by software title and status - var listResp listHostsResponse - s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) - require.Len(t, listResp.Hosts, 1) - require.Equal(t, h.ID, listResp.Hosts[0].ID) - - var countResp countHostsResponse - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) - require.Equal(t, 1, countResp.Count) - - listResp = listHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "failed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) - require.Len(t, listResp.Hosts, 0) - - countResp = countHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "failed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) - require.Equal(t, 0, countResp.Count) - - listResp = listHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "installed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) - require.Len(t, listResp.Hosts, 0) - - countResp = countHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "installed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) - require.Equal(t, 0, countResp.Count) - - var labelResp createLabelResponse - s.DoJSON("POST", "/api/latest/fleet/labels", &createLabelRequest{fleet.LabelPayload{ - Name: "test", - Hosts: []string{h.Hostname}, - }}, http.StatusOK, &labelResp) - require.NotZero(t, labelResp.Label.ID) - - listResp = listHostsResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp) - require.Len(t, listResp.Hosts, 1) - require.Equal(t, h.ID, listResp.Hosts[0].ID) - - countResp = countHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) - require.Equal(t, 1, countResp.Count) - - listResp = listHostsResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) - require.Len(t, listResp.Hosts, 1) - require.Equal(t, h.ID, listResp.Hosts[0].ID) - - countResp = countHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) - require.Equal(t, 1, countResp.Count) - - listResp = listHostsResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", "installed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) - require.Len(t, listResp.Hosts, 0) - - countResp = countHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "installed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) - require.Equal(t, 0, countResp.Count) - - listResp = listHostsResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", "failed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) - require.Len(t, listResp.Hosts, 0) - - countResp = countHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "failed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) - require.Equal(t, 0, countResp.Count) - - // filter validations - r := s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "uninstalled") - require.Contains(t, extractServerErrorText(r.Body), "Invalid software_status") - r = s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "installed") - require.Contains(t, extractServerErrorText(r.Body), "Missing software_title_id") - r = s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "installed", "software_title_id", "1") - require.Contains(t, extractServerErrorText(r.Body), "Missing team_id") - r = s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "installed", "team_id", "1") - require.Contains(t, extractServerErrorText(r.Body), "Missing software_title_id") - r = s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "installed", "team_id", "1", "software_title_id", "1", "software_version_id", "1") - require.Contains(t, extractServerErrorText(r.Body), "Invalid parameters. The combination of software_version_id and software_title_id is not allowed.") - r = s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "installed", "team_id", "1", "software_title_id", "1", "software_id", "1") - require.Contains(t, extractServerErrorText(r.Body), "Invalid parameters. The combination of software_id and software_title_id is not allowed.") - - // TODO(roberto): once we have endpoints to retrieve installers, - // request them using the orbit node key - - // TODO(sarah): test other statuses once we have endpoints to set results via orbit -} - -func (s *integrationMDMTestSuite) TestHostSoftwareInstallResult() { - ctx := context.Background() - t := s.T() - - host := createOrbitEnrolledHost(t, "linux", "", s.ds) - - // create a software installer and some host install requests - payload := &fleet.UploadSoftwareInstallerPayload{ - InstallScript: "install script", - PreInstallQuery: "pre install query", - PostInstallScript: "post install script", - Filename: "ruby.deb", - Title: "ruby", - } - s.uploadSoftwareInstaller(payload, http.StatusOK, "") - titleID := getSoftwareTitleID(t, s.ds, payload.Title, "deb_packages") - - latestInstallUUID := func() string { - var id string - mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { - return sqlx.GetContext(ctx, q, &id, `SELECT execution_id FROM host_software_installs ORDER BY id DESC LIMIT 1`) - }) - return id - } - - // create some install requests for the host - installUUIDs := make([]string, 3) - for i := 0; i < len(installUUIDs); i++ { - resp := installSoftwareResponse{} - s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/hosts/%d/software/install/%d", host.ID, titleID), nil, http.StatusAccepted, &resp) - installUUIDs[i] = latestInstallUUID() - } - - type result struct { - HostID uint - InstallUUID string - Status fleet.SoftwareInstallerStatus - } - checkResults := func(want result) { - var resp getSoftwareInstallResultsResponse - s.DoJSON("GET", "/api/v1/fleet/software/install/results/"+want.InstallUUID, nil, http.StatusOK, &resp) - - assert.Equal(t, want.HostID, resp.Results.HostID) - assert.Equal(t, want.InstallUUID, resp.Results.InstallUUID) - assert.Equal(t, want.Status, resp.Results.Status) - } - - s.Do("POST", "/api/fleet/orbit/software_install/result", - json.RawMessage(fmt.Sprintf(`{ - "orbit_node_key": %q, - "install_uuid": %q, - "pre_install_condition_output": "1", - "install_script_exit_code": 1, - "install_script_output": "failed" - }`, *host.OrbitNodeKey, installUUIDs[0])), - http.StatusNoContent) - checkResults(result{ - HostID: host.ID, - InstallUUID: installUUIDs[0], - Status: fleet.SoftwareInstallerFailed, - }) - wantAct := fleet.ActivityTypeInstalledSoftware{ - HostID: host.ID, - HostDisplayName: host.DisplayName(), - SoftwareTitle: payload.Title, - InstallUUID: installUUIDs[0], - Status: string(fleet.SoftwareInstallerFailed), - } - s.lastActivityMatches(wantAct.ActivityName(), string(jsonMustMarshal(t, wantAct)), 0) - - s.Do("POST", "/api/fleet/orbit/software_install/result", - json.RawMessage(fmt.Sprintf(`{ - "orbit_node_key": %q, - "install_uuid": %q, - "pre_install_condition_output": "" - }`, *host.OrbitNodeKey, installUUIDs[1])), - http.StatusNoContent) - checkResults(result{ - HostID: host.ID, - InstallUUID: installUUIDs[1], - Status: fleet.SoftwareInstallerFailed, - }) - wantAct.InstallUUID = installUUIDs[1] - s.lastActivityOfTypeMatches(wantAct.ActivityName(), string(jsonMustMarshal(t, wantAct)), 0) - - s.Do("POST", "/api/fleet/orbit/software_install/result", - json.RawMessage(fmt.Sprintf(`{ - "orbit_node_key": %q, - "install_uuid": %q, - "pre_install_condition_output": "1", - "install_script_exit_code": 0, - "install_script_output": "success", - "post_install_script_exit_code": 0, - "post_install_script_output": "ok" - }`, *host.OrbitNodeKey, installUUIDs[2])), - http.StatusNoContent) - checkResults(result{ - HostID: host.ID, - InstallUUID: installUUIDs[2], - Status: fleet.SoftwareInstallerInstalled, - }) - wantAct.InstallUUID = installUUIDs[2] - wantAct.Status = string(fleet.SoftwareInstallerInstalled) - lastActID := s.lastActivityOfTypeMatches(wantAct.ActivityName(), string(jsonMustMarshal(t, wantAct)), 0) - - // non-existing installation uuid - s.Do("POST", "/api/fleet/orbit/software_install/result", - json.RawMessage(fmt.Sprintf(`{ - "orbit_node_key": %q, - "install_uuid": "uuid-no-such", - "pre_install_condition_output": "" - }`, *host.OrbitNodeKey)), - http.StatusNotFound) - // no new activity created - s.lastActivityOfTypeMatches(wantAct.ActivityName(), string(jsonMustMarshal(t, wantAct)), lastActID) -} - -func (s *integrationMDMTestSuite) uploadSoftwareInstaller(payload *fleet.UploadSoftwareInstallerPayload, expectedStatus int, expectedError string) { - t := s.T() - openFile := func(name string) *os.File { - f, err := os.Open(filepath.Join("testdata", "software-installers", name)) - require.NoError(t, err) - return f - } - - f := openFile(payload.Filename) - defer f.Close() - - payload.InstallerFile = f - - var b bytes.Buffer - w := multipart.NewWriter(&b) - - // add the software field - fw, err := w.CreateFormFile("software", payload.Filename) - require.NoError(t, err) - n, err := io.Copy(fw, payload.InstallerFile) - require.NoError(t, err) - require.NotZero(t, n) - - // add the team_id field - if payload.TeamID != nil { - require.NoError(t, w.WriteField("team_id", fmt.Sprintf("%d", *payload.TeamID))) - } - // add the remaining fields - require.NoError(t, w.WriteField("install_script", payload.InstallScript)) - require.NoError(t, w.WriteField("pre_install_query", payload.PreInstallQuery)) - require.NoError(t, w.WriteField("post_install_script", payload.PostInstallScript)) - - w.Close() - - headers := map[string]string{ - "Content-Type": w.FormDataContentType(), - "Accept": "application/json", - "Authorization": fmt.Sprintf("Bearer %s", s.token), - } - - r := s.DoRawWithHeaders("POST", "/api/latest/fleet/software/package", b.Bytes(), expectedStatus, headers) - if expectedError != "" { - errMsg := extractServerErrorText(r.Body) - require.Contains(t, errMsg, expectedError) - } -} - -func getSoftwareTitleID(t *testing.T, ds *mysql.Datastore, title, source string) uint { - var id uint - mysql.ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { - return sqlx.GetContext(context.Background(), q, &id, `SELECT id FROM software_titles WHERE name = ? AND source = ? AND browser = ''`, title, source) - }) - return id -} diff --git a/server/service/testing_client.go b/server/service/testing_client.go index 7a182f5652..028a452d91 100644 --- a/server/service/testing_client.go +++ b/server/service/testing_client.go @@ -118,8 +118,11 @@ func (ts *withServer) commonTearDownTest(t *testing.T) { require.NoError(t, ts.ds.DeleteHost(ctx, host.ID)) } - // recalculate software counts will remove the software entries - require.NoError(t, ts.ds.SyncHostsSoftware(context.Background(), time.Now())) + // clean up any software installers + mysql.ExecAdhocSQL(t, ts.ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(ctx, `DELETE FROM software_installers`) + return err + }) lbls, err := ts.ds.ListLabels(ctx, fleet.TeamFilter{}, fleet.ListOptions{}) require.NoError(t, err) @@ -176,9 +179,13 @@ func (ts *withServer) commonTearDownTest(t *testing.T) { require.NoError(t, err) } - // SyncHostsSoftware performs a cleanup. + // Do the software/titles cleanup. err = ts.ds.SyncHostsSoftware(ctx, time.Now()) require.NoError(t, err) + err = ts.ds.ReconcileSoftwareTitles(ctx) + require.NoError(t, err) + err = ts.ds.SyncHostsSoftwareTitles(ctx, time.Now()) + require.NoError(t, err) // delete orphaned scripts mysql.ExecAdhocSQL(t, ts.ds, func(q sqlx.ExtContext) error { From 3a3126235350a0bfb05bdf89da23ca1b179d8b06 Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Tue, 14 May 2024 15:06:33 -0300 Subject: [PATCH 39/56] add CLI and endpoints to set software via fleetctl apply (#18876) for #18325 # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements) - [x] Added/updated tests - [x] Manual QA for all new/changed functionality --------- Co-authored-by: Martin Angers --- cmd/fleetctl/get_test.go | 3 + .../expectedGetConfigAppConfigJson.json | 260 ++++++------ ...ectedGetConfigIncludeServerConfigJson.json | 384 +++++++++--------- .../testdata/expectedGetTeamsJson.json | 2 + .../testdata/expectedGetTeamsYaml.yml | 2 + .../macosSetupExpectedTeam1And2Empty.yml | 2 + .../macosSetupExpectedTeam1And2Set.yml | 2 + .../testdata/macosSetupExpectedTeam1Empty.yml | 1 + .../testdata/macosSetupExpectedTeam1Set.yml | 1 + ee/server/service/software_installers.go | 263 +++++++++--- ee/server/service/teams.go | 4 + pkg/file/file.go | 97 ++++- pkg/file/file_test.go | 27 +- pkg/file/management.go | 31 +- pkg/file/management_test.go | 6 +- pkg/fleethttp/fleethttp.go | 32 ++ server/datastore/mysql/software_installers.go | 138 +++++++ .../mysql/software_installers_test.go | 114 +++++- server/datastore/mysql/software_titles.go | 2 - server/fleet/datastore.go | 6 +- server/fleet/scripts.go | 7 + server/fleet/service.go | 4 + server/fleet/software_installer.go | 13 +- server/fleet/teams.go | 43 +- server/mock/datastore_mock.go | 12 + server/service/client.go | 98 +++++ server/service/client_mdm.go | 27 +- server/service/client_teams.go | 10 + server/service/client_test.go | 23 -- server/service/handler.go | 1 + server/service/integration_enterprise_test.go | 194 +++++++++ server/service/software_installers.go | 34 ++ 32 files changed, 1364 insertions(+), 479 deletions(-) diff --git a/cmd/fleetctl/get_test.go b/cmd/fleetctl/get_test.go index 4b64f3c169..e961ce8e48 100644 --- a/cmd/fleetctl/get_test.go +++ b/cmd/fleetctl/get_test.go @@ -2235,6 +2235,9 @@ func TestGetTeamsYAMLAndApply(t *testing.T) { declaration.DeclarationUUID = uuid.NewString() return declaration, nil } + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + return nil + } actualYaml := runAppForTest(t, []string{"get", "teams", "--yaml"}) yamlFilePath := writeTmpYml(t, actualYaml) diff --git a/cmd/fleetctl/testdata/expectedGetConfigAppConfigJson.json b/cmd/fleetctl/testdata/expectedGetConfigAppConfigJson.json index 5827a3d776..3c8e19f5d9 100644 --- a/cmd/fleetctl/testdata/expectedGetConfigAppConfigJson.json +++ b/cmd/fleetctl/testdata/expectedGetConfigAppConfigJson.json @@ -1,132 +1,132 @@ { - "kind": "config", - "apiVersion": "v1", - "spec": { - "org_info": { - "org_name": "", - "org_logo_url": "", - "org_logo_url_light_background": "", - "contact_url": "https://fleetdm.com/company/contact" - }, - "server_settings": { - "server_url": "", - "live_query_disabled": false, - "query_reports_disabled": false, - "enable_analytics": false, - "deferred_save_host": false, - "scripts_disabled": false, - "ai_features_disabled": false - }, - "smtp_settings": { - "enable_smtp": false, - "configured": false, - "sender_address": "", - "server": "", - "port": 0, - "authentication_type": "", - "user_name": "", - "password": "", - "enable_ssl_tls": false, - "authentication_method": "", - "domain": "", - "verify_ssl_certs": false, - "enable_start_tls": false - }, - "host_expiry_settings": { - "host_expiry_enabled": false, - "host_expiry_window": 0 - }, - "activity_expiry_settings": { - "activity_expiry_enabled": false, - "activity_expiry_window": 0 - }, - "features": { - "enable_host_users": true, - "enable_software_inventory": false - }, - "sso_settings": { - "entity_id": "", - "issuer_uri": "", - "idp_image_url": "", - "metadata": "", - "metadata_url": "", - "idp_name": "", - "enable_jit_provisioning": false, - "enable_jit_role_sync": false, - "enable_sso": false, - "enable_sso_idp_login": false - }, - "fleet_desktop": { - "transparency_url": "https://fleetdm.com/transparency" - }, - "vulnerability_settings": { - "databases_path": "/some/path" - }, - "webhook_settings": { - "host_status_webhook": { - "enable_host_status_webhook": false, - "destination_url": "", - "host_percentage": 0, - "days_count": 0 - }, - "failing_policies_webhook": { - "enable_failing_policies_webhook": false, - "destination_url": "", - "policy_ids": null, - "host_batch_size": 0 - }, - "vulnerabilities_webhook": { - "enable_vulnerabilities_webhook": false, - "destination_url": "", - "host_batch_size": 0 - }, - "interval": "0s" - }, - "integrations": { - "jira": null, - "zendesk": null, - "google_calendar": null - }, - "mdm": { - "apple_bm_terms_expired": false, - "apple_bm_enabled_and_configured": false, - "enabled_and_configured": false, - "apple_bm_default_team": "", - "windows_enabled_and_configured": false, - "enable_disk_encryption": false, - "macos_updates": { - "minimum_version": null, - "deadline": null - }, - "windows_updates": { - "deadline_days": 7, - "grace_period_days": 3 - }, - "macos_migration": { - "enable": false, - "mode": "", - "webhook_url": "" - }, - "macos_settings": { - "custom_settings": null - }, - "macos_setup": { - "bootstrap_package": null, - "enable_end_user_authentication": false, - "macos_setup_assistant": null, - "enable_release_device_manually": false - }, - "windows_settings": { - "custom_settings": null - }, - "end_user_authentication": { - "entity_id": "", - "issuer_uri": "", - "metadata": "", - "metadata_url": "", - "idp_name": "" - } - }, - "scripts": null - } + "kind": "config", + "apiVersion": "v1", + "spec": { + "org_info": { + "org_name": "", + "org_logo_url": "", + "org_logo_url_light_background": "", + "contact_url": "https://fleetdm.com/company/contact" + }, + "server_settings": { + "server_url": "", + "live_query_disabled": false, + "query_reports_disabled": false, + "enable_analytics": false, + "deferred_save_host": false, + "scripts_disabled": false, + "ai_features_disabled": false + }, + "smtp_settings": { + "enable_smtp": false, + "configured": false, + "sender_address": "", + "server": "", + "port": 0, + "authentication_type": "", + "user_name": "", + "password": "", + "enable_ssl_tls": false, + "authentication_method": "", + "domain": "", + "verify_ssl_certs": false, + "enable_start_tls": false + }, + "host_expiry_settings": { + "host_expiry_enabled": false, + "host_expiry_window": 0 + }, + "activity_expiry_settings": { + "activity_expiry_enabled": false, + "activity_expiry_window": 0 + }, + "features": { + "enable_host_users": true, + "enable_software_inventory": false + }, + "sso_settings": { + "entity_id": "", + "issuer_uri": "", + "idp_image_url": "", + "metadata": "", + "metadata_url": "", + "idp_name": "", + "enable_jit_provisioning": false, + "enable_jit_role_sync": false, + "enable_sso": false, + "enable_sso_idp_login": false + }, + "fleet_desktop": { + "transparency_url": "https://fleetdm.com/transparency" + }, + "vulnerability_settings": { + "databases_path": "/some/path" + }, + "webhook_settings": { + "host_status_webhook": { + "enable_host_status_webhook": false, + "destination_url": "", + "host_percentage": 0, + "days_count": 0 + }, + "failing_policies_webhook": { + "enable_failing_policies_webhook": false, + "destination_url": "", + "policy_ids": null, + "host_batch_size": 0 + }, + "vulnerabilities_webhook": { + "enable_vulnerabilities_webhook": false, + "destination_url": "", + "host_batch_size": 0 + }, + "interval": "0s" + }, + "integrations": { + "jira": null, + "zendesk": null, + "google_calendar": null + }, + "mdm": { + "apple_bm_terms_expired": false, + "apple_bm_enabled_and_configured": false, + "enabled_and_configured": false, + "apple_bm_default_team": "", + "windows_enabled_and_configured": false, + "enable_disk_encryption": false, + "macos_updates": { + "minimum_version": null, + "deadline": null + }, + "windows_updates": { + "deadline_days": 7, + "grace_period_days": 3 + }, + "macos_migration": { + "enable": false, + "mode": "", + "webhook_url": "" + }, + "macos_settings": { + "custom_settings": null + }, + "macos_setup": { + "bootstrap_package": null, + "enable_end_user_authentication": false, + "macos_setup_assistant": null, + "enable_release_device_manually": false + }, + "windows_settings": { + "custom_settings": null + }, + "end_user_authentication": { + "entity_id": "", + "issuer_uri": "", + "metadata": "", + "metadata_url": "", + "idp_name": "" + } + }, + "scripts": null + } } diff --git a/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigJson.json b/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigJson.json index 5df52f4c8e..b66af1c6df 100644 --- a/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigJson.json +++ b/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigJson.json @@ -1,194 +1,194 @@ { - "kind": "config", - "apiVersion": "v1", - "spec": { - "org_info": { - "org_name": "", - "org_logo_url": "", - "org_logo_url_light_background": "", - "contact_url": "https://fleetdm.com/company/contact" - }, - "server_settings": { - "server_url": "", - "live_query_disabled": false, - "query_reports_disabled": false, - "enable_analytics": false, - "deferred_save_host": false, - "scripts_disabled": false, - "ai_features_disabled": false - }, - "smtp_settings": { - "enable_smtp": false, - "configured": false, - "sender_address": "", - "server": "", - "port": 0, - "authentication_type": "", - "user_name": "", - "password": "", - "enable_ssl_tls": false, - "authentication_method": "", - "domain": "", - "verify_ssl_certs": false, - "enable_start_tls": false - }, - "host_expiry_settings": { - "host_expiry_enabled": false, - "host_expiry_window": 0 - }, - "activity_expiry_settings": { - "activity_expiry_enabled": false, - "activity_expiry_window": 0 - }, - "features": { - "enable_host_users": true, - "enable_software_inventory": false - }, - "mdm": { - "apple_bm_default_team": "", - "apple_bm_terms_expired": false, - "apple_bm_enabled_and_configured": false, - "enabled_and_configured": false, - "windows_enabled_and_configured": false, - "enable_disk_encryption": false, - "macos_updates": { - "minimum_version": null, - "deadline": null - }, - "windows_updates": { - "deadline_days": 7, - "grace_period_days": 3 - }, - "macos_migration": { - "enable": false, - "mode": "", - "webhook_url": "" - }, - "macos_settings": { - "custom_settings": null - }, - "macos_setup": { - "bootstrap_package": null, - "enable_end_user_authentication": false, - "macos_setup_assistant": null, - "enable_release_device_manually": false - }, - "windows_settings": { - "custom_settings": null - }, - "end_user_authentication": { - "entity_id": "", - "issuer_uri": "", - "metadata": "", - "metadata_url": "", - "idp_name": "" - } - }, - "scripts": null, - "sso_settings": { - "enable_jit_provisioning": false, - "enable_jit_role_sync": false, - "entity_id": "", - "issuer_uri": "", - "idp_image_url": "", - "metadata": "", - "metadata_url": "", - "idp_name": "", - "enable_sso": false, - "enable_sso_idp_login": false - }, - "fleet_desktop": { - "transparency_url": "https://fleetdm.com/transparency" - }, - "vulnerability_settings": { - "databases_path": "/some/path" - }, - "webhook_settings": { - "host_status_webhook": { - "enable_host_status_webhook": false, - "destination_url": "", - "host_percentage": 0, - "days_count": 0 - }, - "failing_policies_webhook": { - "enable_failing_policies_webhook": false, - "destination_url": "", - "policy_ids": null, - "host_batch_size": 0 - }, - "vulnerabilities_webhook": { - "enable_vulnerabilities_webhook": false, - "destination_url": "", - "host_batch_size": 0 - }, - "interval": "0s" - }, - "integrations": { - "jira": null, - "zendesk": null, - "google_calendar": null - }, - "update_interval": { - "osquery_detail": "1h0m0s", - "osquery_policy": "1h0m0s" - }, - "vulnerabilities": { - "databases_path": "", - "periodicity": "0s", - "cpe_database_url": "", - "cpe_translations_url": "", - "cve_feed_prefix_url": "", - "current_instance_checks": "", - "disable_data_sync": false, - "recent_vulnerability_max_age": "0s", - "disable_win_os_vulnerabilities": false - }, - "license": { - "tier": "free", - "expiration": "0001-01-01T00:00:00Z" - }, - "logging": { - "debug": true, - "json": false, - "result": { - "plugin": "filesystem", - "config": { - "enable_log_compression": false, - "enable_log_rotation": false, - "result_log_file": "/dev/null", - "status_log_file": "/dev/null", - "audit_log_file": "/dev/null", - "max_size": 500, - "max_age": 0, - "max_backups": 0 - } - }, - "status": { - "plugin": "filesystem", - "config": { - "enable_log_compression": false, - "enable_log_rotation": false, - "result_log_file": "/dev/null", - "status_log_file": "/dev/null", - "audit_log_file": "/dev/null", - "max_size": 500, - "max_age": 0, - "max_backups": 0 - } - }, - "audit": { - "plugin": "filesystem", - "config": { - "enable_log_compression": false, - "enable_log_rotation": false, - "result_log_file": "/dev/null", - "status_log_file": "/dev/null", - "audit_log_file": "/dev/null", - "max_size": 500, - "max_age": 0, - "max_backups": 0 - } - } - } - } + "kind": "config", + "apiVersion": "v1", + "spec": { + "org_info": { + "org_name": "", + "org_logo_url": "", + "org_logo_url_light_background": "", + "contact_url": "https://fleetdm.com/company/contact" + }, + "server_settings": { + "server_url": "", + "live_query_disabled": false, + "query_reports_disabled": false, + "enable_analytics": false, + "deferred_save_host": false, + "scripts_disabled": false, + "ai_features_disabled": false + }, + "smtp_settings": { + "enable_smtp": false, + "configured": false, + "sender_address": "", + "server": "", + "port": 0, + "authentication_type": "", + "user_name": "", + "password": "", + "enable_ssl_tls": false, + "authentication_method": "", + "domain": "", + "verify_ssl_certs": false, + "enable_start_tls": false + }, + "host_expiry_settings": { + "host_expiry_enabled": false, + "host_expiry_window": 0 + }, + "activity_expiry_settings": { + "activity_expiry_enabled": false, + "activity_expiry_window": 0 + }, + "features": { + "enable_host_users": true, + "enable_software_inventory": false + }, + "mdm": { + "apple_bm_default_team": "", + "apple_bm_terms_expired": false, + "apple_bm_enabled_and_configured": false, + "enabled_and_configured": false, + "windows_enabled_and_configured": false, + "enable_disk_encryption": false, + "macos_updates": { + "minimum_version": null, + "deadline": null + }, + "windows_updates": { + "deadline_days": 7, + "grace_period_days": 3 + }, + "macos_migration": { + "enable": false, + "mode": "", + "webhook_url": "" + }, + "macos_settings": { + "custom_settings": null + }, + "macos_setup": { + "bootstrap_package": null, + "enable_end_user_authentication": false, + "macos_setup_assistant": null, + "enable_release_device_manually": false + }, + "windows_settings": { + "custom_settings": null + }, + "end_user_authentication": { + "entity_id": "", + "issuer_uri": "", + "metadata": "", + "metadata_url": "", + "idp_name": "" + } + }, + "scripts": null, + "sso_settings": { + "enable_jit_provisioning": false, + "enable_jit_role_sync": false, + "entity_id": "", + "issuer_uri": "", + "idp_image_url": "", + "metadata": "", + "metadata_url": "", + "idp_name": "", + "enable_sso": false, + "enable_sso_idp_login": false + }, + "fleet_desktop": { + "transparency_url": "https://fleetdm.com/transparency" + }, + "vulnerability_settings": { + "databases_path": "/some/path" + }, + "webhook_settings": { + "host_status_webhook": { + "enable_host_status_webhook": false, + "destination_url": "", + "host_percentage": 0, + "days_count": 0 + }, + "failing_policies_webhook": { + "enable_failing_policies_webhook": false, + "destination_url": "", + "policy_ids": null, + "host_batch_size": 0 + }, + "vulnerabilities_webhook": { + "enable_vulnerabilities_webhook": false, + "destination_url": "", + "host_batch_size": 0 + }, + "interval": "0s" + }, + "integrations": { + "jira": null, + "zendesk": null, + "google_calendar": null + }, + "update_interval": { + "osquery_detail": "1h0m0s", + "osquery_policy": "1h0m0s" + }, + "vulnerabilities": { + "databases_path": "", + "periodicity": "0s", + "cpe_database_url": "", + "cpe_translations_url": "", + "cve_feed_prefix_url": "", + "current_instance_checks": "", + "disable_data_sync": false, + "recent_vulnerability_max_age": "0s", + "disable_win_os_vulnerabilities": false + }, + "license": { + "tier": "free", + "expiration": "0001-01-01T00:00:00Z" + }, + "logging": { + "debug": true, + "json": false, + "result": { + "plugin": "filesystem", + "config": { + "enable_log_compression": false, + "enable_log_rotation": false, + "result_log_file": "/dev/null", + "status_log_file": "/dev/null", + "audit_log_file": "/dev/null", + "max_size": 500, + "max_age": 0, + "max_backups": 0 + } + }, + "status": { + "plugin": "filesystem", + "config": { + "enable_log_compression": false, + "enable_log_rotation": false, + "result_log_file": "/dev/null", + "status_log_file": "/dev/null", + "audit_log_file": "/dev/null", + "max_size": 500, + "max_age": 0, + "max_backups": 0 + } + }, + "audit": { + "plugin": "filesystem", + "config": { + "enable_log_compression": false, + "enable_log_rotation": false, + "result_log_file": "/dev/null", + "status_log_file": "/dev/null", + "audit_log_file": "/dev/null", + "max_size": 500, + "max_age": 0, + "max_backups": 0 + } + } + } + } } diff --git a/cmd/fleetctl/testdata/expectedGetTeamsJson.json b/cmd/fleetctl/testdata/expectedGetTeamsJson.json index ad84fe3d91..1af52bee61 100644 --- a/cmd/fleetctl/testdata/expectedGetTeamsJson.json +++ b/cmd/fleetctl/testdata/expectedGetTeamsJson.json @@ -53,6 +53,7 @@ } }, "scripts": null, + "software": null, "user_count": 99, "host_count": 42 } @@ -128,6 +129,7 @@ } }, "scripts": null, + "software": null, "user_count": 87, "host_count": 43 } diff --git a/cmd/fleetctl/testdata/expectedGetTeamsYaml.yml b/cmd/fleetctl/testdata/expectedGetTeamsYaml.yml index 1b04f6e599..f1315fcf24 100644 --- a/cmd/fleetctl/testdata/expectedGetTeamsYaml.yml +++ b/cmd/fleetctl/testdata/expectedGetTeamsYaml.yml @@ -29,6 +29,7 @@ spec: enable_release_device_manually: false macos_setup_assistant: scripts: null + software: null webhook_settings: host_status_webhook: null name: team1 @@ -72,6 +73,7 @@ spec: enable_release_device_manually: false macos_setup_assistant: scripts: null + software: null webhook_settings: host_status_webhook: null name: team2 diff --git a/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Empty.yml b/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Empty.yml index e26318d031..28f815240d 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Empty.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Empty.yml @@ -29,6 +29,7 @@ spec: deadline_days: null grace_period_days: null scripts: null + software: null webhook_settings: host_status_webhook: null name: tm1 @@ -62,6 +63,7 @@ spec: deadline_days: null grace_period_days: null scripts: null + software: null webhook_settings: host_status_webhook: null name: tm2 diff --git a/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Set.yml b/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Set.yml index 7f9da95cda..ef911ec34f 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Set.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Set.yml @@ -29,6 +29,7 @@ spec: deadline_days: null grace_period_days: null scripts: null + software: null webhook_settings: host_status_webhook: null name: tm1 @@ -62,6 +63,7 @@ spec: deadline_days: null grace_period_days: null scripts: null + software: null webhook_settings: host_status_webhook: null name: tm2 diff --git a/cmd/fleetctl/testdata/macosSetupExpectedTeam1Empty.yml b/cmd/fleetctl/testdata/macosSetupExpectedTeam1Empty.yml index 7bbf81022c..19f92edbc0 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedTeam1Empty.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedTeam1Empty.yml @@ -29,6 +29,7 @@ spec: windows_settings: custom_settings: null scripts: null + software: null webhook_settings: host_status_webhook: null name: tm1 diff --git a/cmd/fleetctl/testdata/macosSetupExpectedTeam1Set.yml b/cmd/fleetctl/testdata/macosSetupExpectedTeam1Set.yml index 10a3d36b13..9862a2d66d 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedTeam1Set.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedTeam1Set.yml @@ -28,6 +28,7 @@ spec: deadline_days: null grace_period_days: null scripts: null + software: null webhook_settings: host_status_webhook: null name: tm1 diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index 1543ffc950..e642b414c2 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -1,83 +1,48 @@ package service import ( + "bytes" "context" "encoding/hex" "errors" "fmt" + "io" + "mime" "net/http" + "net/url" "path/filepath" - "strings" "github.com/fleetdm/fleet/v4/pkg/file" + "github.com/fleetdm/fleet/v4/pkg/fleethttp" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" hostctx "github.com/fleetdm/fleet/v4/server/contexts/host" "github.com/fleetdm/fleet/v4/server/contexts/viewer" "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/go-kit/kit/log/level" + "github.com/go-kit/log/level" + "golang.org/x/sync/errgroup" ) func (svc *Service) UploadSoftwareInstaller(ctx context.Context, payload *fleet.UploadSoftwareInstallerPayload) error { if err := svc.authz.Authorize(ctx, &fleet.SoftwareInstaller{TeamID: payload.TeamID}, fleet.ActionWrite); err != nil { return err } - if payload == nil { - return ctxerr.New(ctx, "payload is required") - } - - if payload.InstallerFile == nil { - return ctxerr.New(ctx, "installer file is required") - } vc, ok := viewer.FromContext(ctx) if !ok { return fleet.ErrNoContext } - title, vers, hash, err := file.ExtractInstallerMetadata(payload.Filename, payload.InstallerFile) - if err != nil { - // TODO: confirm error handling - if strings.Contains(err.Error(), "unsupported file type") { - return &fleet.BadRequestError{ - Message: "The file should be .pkg, .msi, .exe or .deb.", - InternalErr: ctxerr.Wrap(ctx, err, "extracting metadata from installer"), - } - } - return ctxerr.Wrap(ctx, err, "extracting metadata from installer") - } - payload.Title = title - payload.Version = vers - payload.StorageID = hex.EncodeToString(hash) - - // checck if exists in the installer store - exists, err := svc.softwareInstallStore.Exists(ctx, payload.StorageID) - if err != nil { - return ctxerr.Wrap(ctx, err, "checking if installer exists") - } - if !exists { - // reset the reader before storing (it was consumed to extract metadata) - if _, err := payload.InstallerFile.Seek(0, 0); err != nil { - return ctxerr.Wrap(ctx, err, "resetting installer file reader") - } - if err := svc.softwareInstallStore.Put(ctx, payload.StorageID, payload.InstallerFile); err != nil { - return ctxerr.Wrap(ctx, err, "storing installer") - } + if _, err := svc.addMetadataToSoftwarePayload(ctx, payload); err != nil { + return ctxerr.Wrap(ctx, err, "adding metadata to payload") } - if payload.InstallScript == "" { - payload.InstallScript = file.GetInstallScript(payload.Filename) + if err := svc.storeSoftware(ctx, payload); err != nil { + return ctxerr.Wrap(ctx, err, "storing software installer") } // TODO: basic validation of install and post-install script (e.g., supported interpreters)? - // TODO: any validation of pre-install query? - source, err := fleet.SofwareInstallerSourceFromFilename(payload.Filename) - if err != nil { - return ctxerr.Wrap(ctx, err, "determining source from filename") - } - payload.Source = source - installerID, err := svc.ds.MatchOrCreateSoftwareInstaller(ctx, payload) if err != nil { return ctxerr.Wrap(ctx, err, "matching or creating software installer") @@ -97,7 +62,7 @@ func (svc *Service) UploadSoftwareInstaller(ctx context.Context, payload *fleet. // Create activity if err := svc.ds.NewActivity(ctx, vc.User, fleet.ActivityTypeAddedSoftware{ - SoftwareTitle: title, + SoftwareTitle: payload.Title, SoftwarePackage: payload.Filename, TeamName: teamName, TeamID: payload.TeamID, @@ -318,3 +283,207 @@ func (svc *Service) GetSoftwareInstallResults(ctx context.Context, resultUUID st return res, nil } + +func (svc *Service) storeSoftware(ctx context.Context, payload *fleet.UploadSoftwareInstallerPayload) error { + // check if exists in the installer store + exists, err := svc.softwareInstallStore.Exists(ctx, payload.StorageID) + if err != nil { + return ctxerr.Wrap(ctx, err, "checking if installer exists") + } + if !exists { + if err := svc.softwareInstallStore.Put(ctx, payload.StorageID, payload.InstallerFile); err != nil { + return ctxerr.Wrap(ctx, err, "storing installer") + } + } + + return nil +} + +func (svc *Service) addMetadataToSoftwarePayload(ctx context.Context, payload *fleet.UploadSoftwareInstallerPayload) (extension string, err error) { + if payload == nil { + return "", ctxerr.New(ctx, "payload is required") + } + + if payload.InstallerFile == nil { + return "", ctxerr.New(ctx, "installer file is required") + } + + title, vers, ext, hash, err := file.ExtractInstallerMetadata(payload.InstallerFile) + if err != nil { + if errors.Is(err, file.ErrUnsupportedType) { + return "", &fleet.BadRequestError{ + Message: "The file should be .pkg, .msi, .exe or .deb.", + InternalErr: ctxerr.Wrap(ctx, err, "extracting metadata from installer"), + } + } + return "", ctxerr.Wrap(ctx, err, "extracting metadata from installer") + } + payload.Title = title + payload.Version = vers + payload.StorageID = hex.EncodeToString(hash) + + // reset the reader (it was consumed to extract metadata) + if _, err := payload.InstallerFile.Seek(0, 0); err != nil { + return "", ctxerr.Wrap(ctx, err, "resetting installer file reader") + } + + if payload.InstallScript == "" { + payload.InstallScript = file.GetInstallScript(ext) + } + + source, err := fleet.SofwareInstallerSourceFromExtension(ext) + if err != nil { + return "", ctxerr.Wrap(ctx, err, "determining source from extension") + } + payload.Source = source + + return ext, nil +} + +const maxInstallerSizeBytes int64 = 1024 * 1024 * 500 + +func (svc *Service) BatchSetSoftwareInstallers(ctx context.Context, tmName string, payloads []fleet.SoftwareInstallerPayload, dryRun bool) error { + if tmName == "" { + svc.authz.SkipAuthorization(ctx) // so that the error message is not replaced by "forbidden" + return ctxerr.Wrap(ctx, fleet.NewInvalidArgumentError("team_name", "must not be empty")) + } + + if err := svc.authz.Authorize(ctx, &fleet.Team{}, fleet.ActionRead); err != nil { + return err + } + + tm, err := svc.ds.TeamByName(ctx, tmName) + if err != nil { + // If this is a dry run, the team may not have been created yet + if dryRun && fleet.IsNotFound(err) { + return nil + } + return err + } + + if err := svc.authz.Authorize(ctx, &fleet.SoftwareInstaller{TeamID: &tm.ID}, fleet.ActionWrite); err != nil { + return ctxerr.Wrap(ctx, err, "validating authorization") + } + + g, workerCtx := errgroup.WithContext(ctx) + g.SetLimit(3) + // critical to avoid data race, the slice is pre-allocated and each + // goroutine only writes to its index. + installers := make([]*fleet.UploadSoftwareInstallerPayload, len(payloads)) + + client := fleethttp.NewClient() + client.Transport = fleethttp.NewSizeLimitTransport(maxInstallerSizeBytes) + for i, p := range payloads { + i, p := i, p + + g.Go(func() error { + // validate the URL before doing the request + _, err := url.ParseRequestURI(p.URL) + if err != nil { + return fleet.NewInvalidArgumentError( + "software.url", + fmt.Sprintf("Couldn't edit software. URL (%q) is invalid", p.URL), + ) + } + + req, err := http.NewRequestWithContext(workerCtx, http.MethodGet, p.URL, nil) + if err != nil { + return ctxerr.Wrapf(ctx, err, "creating request for URL %s", p.URL) + } + + resp, err := client.Do(req) + if err != nil { + var maxBytesErr *http.MaxBytesError + if errors.Is(err, fleethttp.ErrMaxSizeExceeded) || errors.As(err, &maxBytesErr) { + return fleet.NewInvalidArgumentError( + "software.url", + fmt.Sprintf("Couldn't edit software. URL (%q). The maximum file size is %d MB", p.URL, maxInstallerSizeBytes/(1024*1024)), + ) + } + + return ctxerr.Wrapf(ctx, err, "performing request for URL %s", p.URL) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return fleet.NewInvalidArgumentError( + "software.url", + fmt.Sprintf("Couldn't edit software. URL (%q) doesn't exist. Please make sure that URLs are publicy accessible to the internet.", p.URL), + ) + } + + // Allow all 2xx and 3xx status codes in this pass. + if resp.StatusCode > 400 { + return fleet.NewInvalidArgumentError( + "software.url", + fmt.Sprintf("Couldn't edit software. URL (%q) received response status code %d.", p.URL, resp.StatusCode), + ) + } + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return ctxerr.Wrapf(ctx, err, "reading installer %q contents", p.URL) + } + + installer := &fleet.UploadSoftwareInstallerPayload{ + TeamID: &tm.ID, + InstallScript: p.InstallScript, + PreInstallQuery: p.PreInstallQuery, + PostInstallScript: p.PostInstallScript, + InstallerFile: bytes.NewReader(bodyBytes), + } + + ext, err := svc.addMetadataToSoftwarePayload(ctx, installer) + if err != nil { + return err + } + + var filename string + cdh, ok := resp.Header["Content-Disposition"] + if ok && len(cdh) > 0 { + _, params, err := mime.ParseMediaType(cdh[0]) + if err == nil { + filename = params["filename"] + } + } + // if it fails, try to extract it from the URL + if filename == "" { + filename = file.ExtractFilenameFromURLPath(p.URL, ext) + } + // if empty, resort to a default name + if filename == "" { + filename = fmt.Sprintf("package.%s", ext) + } + installer.Filename = filename + + installers[i] = installer + + return nil + }) + } + + if err := g.Wait(); err != nil { + // NOTE: intentionally not wrapping to avoid polluting user + // errors. + return err + } + + if dryRun { + return nil + } + + for _, payload := range installers { + if err := svc.storeSoftware(ctx, payload); err != nil { + return ctxerr.Wrap(ctx, err, "storing software installer") + } + } + + if err := svc.ds.BatchSetSoftwareInstallers(ctx, &tm.ID, installers); err != nil { + return ctxerr.Wrap(ctx, err, "batch set software installers") + } + + // Note: per @noahtalerman we don't want activity items for CLI actions + // anymore, so that's intentionally skipped. + + return nil +} diff --git a/ee/server/service/teams.go b/ee/server/service/teams.go index 25f9f5d40e..b19c735468 100644 --- a/ee/server/service/teams.go +++ b/ee/server/service/teams.go @@ -1085,6 +1085,10 @@ func (svc *Service) editTeamFromSpec( team.Config.Scripts = spec.Scripts } + if spec.Software.Set { + team.Config.Software = spec.Software + } + if len(secrets) > 0 { team.Secrets = secrets } diff --git a/pkg/file/file.go b/pkg/file/file.go index 833cc88739..c4fecc6fbd 100644 --- a/pkg/file/file.go +++ b/pkg/file/file.go @@ -1,33 +1,78 @@ package file import ( + "bufio" + "bytes" + "encoding/binary" "errors" "fmt" "io" "io/fs" + "net/url" "os" + "path" "path/filepath" "github.com/fleetdm/fleet/v4/pkg/secure" ) +var ErrUnsupportedType = errors.New("unsupported file type") + // ExtractInstallerMetadata extracts the software name and version from the // installer file and returns them along with the sha256 hash of the bytes. The -// format of the installer is determined based on the extension of the -// filename. -func ExtractInstallerMetadata(filename string, r io.Reader) (name, version string, shaSum []byte, err error) { - switch ext := filepath.Ext(filename); ext { - case ".deb": - return ExtractDebMetadata(r) - case ".exe": - return ExtractPEMetadata(r) - case ".pkg": - return ExtractXARMetadata(r) - case ".msi": - return ExtractMSIMetadata(r) - default: - return "", "", nil, fmt.Errorf("unsupported file type: %s", ext) +// format of the installer is determined based on the magic bytes of the content. +func ExtractInstallerMetadata(r io.Reader) (name, version, extension string, shaSum []byte, err error) { + br := bufio.NewReader(r) + extension, err = typeFromBytes(br) + if err != nil { + return "", "", "", nil, err } + + switch extension { + case "deb": + name, version, shaSum, err = ExtractDebMetadata(br) + case "exe": + name, version, shaSum, err = ExtractPEMetadata(br) + case "pkg": + name, version, shaSum, err = ExtractXARMetadata(br) + case "msi": + name, version, shaSum, err = ExtractMSIMetadata(br) + default: + return "", "", "", nil, ErrUnsupportedType + } + + return name, version, extension, shaSum, err +} + +func typeFromBytes(br *bufio.Reader) (string, error) { + switch { + case hasPrefix(br, []byte{0x78, 0x61, 0x72, 0x21}): + return "pkg", nil + case hasPrefix(br, []byte("!\ndebian")): + return "deb", nil + case hasPrefix(br, []byte{0xd0, 0xcf}): + return "msi", nil + case hasPrefix(br, []byte("MZ")): + if blob, _ := br.Peek(0x3e); len(blob) == 0x3e { + reloc := binary.LittleEndian.Uint16(blob[0x3c:0x3e]) + if blob, err := br.Peek(int(reloc) + 4); err == nil { + if bytes.Equal(blob[reloc:reloc+4], []byte("PE\x00\x00")) { + return "exe", nil + } + } + } + fallthrough + default: + return "", ErrUnsupportedType + } +} + +func hasPrefix(br *bufio.Reader, blob []byte) bool { + d, _ := br.Peek(len(blob)) + if len(d) < len(blob) { + return false + } + return bytes.Equal(d, blob) } // Copy copies the file from srcPath to dstPath, using the provided permissions. @@ -84,3 +129,27 @@ func Exists(path string) (bool, error) { return info.Mode().IsRegular(), nil } + +func ExtractFilenameFromURLPath(p string, defaultExtension string) string { + u, err := url.Parse(p) + if err != nil { + return "" + } + + invalid := map[string]struct{}{ + "": {}, + ".": {}, + "/": {}, + } + + b := path.Base(u.Path) + if _, ok := invalid[b]; ok { + return "" + } + + if _, ok := invalid[path.Ext(b)]; ok { + return fmt.Sprintf("%s.%s", b, defaultExtension) + } + + return b +} diff --git a/pkg/file/file_test.go b/pkg/file/file_test.go index 231031ccff..aa4fe2b6a7 100644 --- a/pkg/file/file_test.go +++ b/pkg/file/file_test.go @@ -124,16 +124,41 @@ func TestExtractInstallerMetadata(t *testing.T) { t.Fatalf("invalid filename, expected at least 3 sections, got %d: %s", len(parts), dent.Name()) } wantName, wantVersion, wantHash := parts[0], parts[1], parts[2] + wantExtension := filepath.Ext(wantName) f, err := os.Open(filepath.Join("testdata", "installers", dent.Name())) require.NoError(t, err) defer f.Close() - name, version, hash, err := file.ExtractInstallerMetadata(dent.Name(), f) + name, version, ext, hash, err := file.ExtractInstallerMetadata(f) require.NoError(t, err) assert.Equal(t, wantName, name) assert.Equal(t, wantVersion, version) assert.Equal(t, wantHash, hex.EncodeToString(hash)) + assert.Equal(t, wantExtension, ext) }) } } + +func TestExtractFilenameFromURLPath(t *testing.T) { + cases := []struct { + in string + out string + }{ + {"http://example.com", ""}, + {"http://example.com/", ""}, + {"http://example.com?foo=bar", ""}, + {"http://example.com/foo.pkg", "foo.pkg"}, + {"http://example.com/foo.exe", "foo.exe"}, + {"http://example.com/foo.pkg?bar=baz", "foo.pkg"}, + {"http://example.com/foo.bar.pkg", "foo.bar.pkg"}, + {"http://example.com/foo", "foo.pkg"}, + {"http://example.com/foo/bar/baz", "baz.pkg"}, + {"http://example.com/foo?bar=baz", "foo.pkg"}, + } + + for _, c := range cases { + got := file.ExtractFilenameFromURLPath(c.in, "pkg") + require.Equalf(t, c.out, got, "for URL %s", c.in) + } +} diff --git a/pkg/file/management.go b/pkg/file/management.go index dfd53b40ec..f5f981d840 100644 --- a/pkg/file/management.go +++ b/pkg/file/management.go @@ -2,7 +2,6 @@ package file import ( _ "embed" - "path/filepath" ) //go:embed scripts/install_pkg.sh @@ -17,16 +16,17 @@ var installExeScript string //go:embed scripts/install_deb.sh var installDebScript string -// GetInstallScript returns a script that can be used to install the given file -func GetInstallScript(filename string) string { - switch ext := filepath.Ext(filename); ext { - case ".msi": +// GetInstallScript returns a script that can be used to install the +// the given extension +func GetInstallScript(extension string) string { + switch extension { + case "msi": return installMsiScript - case ".deb": + case "deb": return installDebScript - case ".pkg": + case "pkg": return installPkgScript - case ".exe": + case "exe": return installExeScript default: return "" @@ -45,16 +45,17 @@ var removeMsiScript string //go:embed scripts/remove_deb.sh var removeDebScript string -// GetRemoveScript returns a script that can be used to remove the given file -func GetRemoveScript(filename string) string { - switch ext := filepath.Ext(filename); ext { - case ".msi": +// GetRemoveScript returns a script that can be used to remove an +// installer with the given extension. +func GetRemoveScript(extension string) string { + switch extension { + case "msi": return removeMsiScript - case ".deb": + case "deb": return removeDebScript - case ".pkg": + case "pkg": return removePkgScript - case ".exe": + case "exe": return removeExeScript default: return "" diff --git a/pkg/file/management_test.go b/pkg/file/management_test.go index 59f63d1675..422bb637a6 100644 --- a/pkg/file/management_test.go +++ b/pkg/file/management_test.go @@ -43,12 +43,10 @@ func TestGetInstallAndRemoveScript(t *testing.T) { } for itype, scripts := range scriptsByType { - installerPath := "./foo/bar baz." + itype - - gotScript := GetInstallScript(installerPath) + gotScript := GetInstallScript(itype) assertGoldenMatches(t, scripts[0], gotScript, *update) - gotScript = GetRemoveScript(installerPath) + gotScript = GetRemoveScript(itype) assertGoldenMatches(t, scripts[1], gotScript, *update) } } diff --git a/pkg/fleethttp/fleethttp.go b/pkg/fleethttp/fleethttp.go index 2265069532..f3fa0cd248 100644 --- a/pkg/fleethttp/fleethttp.go +++ b/pkg/fleethttp/fleethttp.go @@ -5,6 +5,7 @@ package fleethttp import ( "context" "crypto/tls" + "errors" "fmt" "net/http" "net/url" @@ -147,3 +148,34 @@ func HostnamesMatch(a, b string) (bool, error) { return ap.Hostname() == bp.Hostname(), nil } + +type SizeLimitTransport struct { + maxSizeBytes int64 +} + +var ErrMaxSizeExceeded = errors.New("response body exceeds max size") + +func NewSizeLimitTransport(maxSizeBytes int64) *SizeLimitTransport { + return &SizeLimitTransport{ + maxSizeBytes: maxSizeBytes, + } +} + +func (t *SizeLimitTransport) RoundTrip(req *http.Request) (*http.Response, error) { + resp, err := http.DefaultTransport.RoundTrip(req) + if err != nil { + return nil, err + } + + if contentLen := resp.ContentLength; contentLen > t.maxSizeBytes { + resp.Body.Close() + return nil, ErrMaxSizeExceeded + } + + // if no Content-Length header, limit reading the body + if resp.ContentLength < 0 { + resp.Body = http.MaxBytesReader(nil, resp.Body, t.maxSizeBytes) + } + + return resp, nil +} diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 8895411e9a..40777f2354 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "github.com/fleetdm/fleet/v4/server/authz" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" @@ -445,3 +446,140 @@ func (ds *Datastore) CleanupUnusedSoftwareInstallers(ctx context.Context, softwa _, err := softwareInstallStore.Cleanup(ctx, storageIDs) return ctxerr.Wrap(ctx, err, "cleanup unused software installers") } + +func (ds *Datastore) BatchSetSoftwareInstallers(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + const upsertSoftwareTitles = ` +INSERT INTO software_titles + (name, source, browser) +VALUES + %s +ON DUPLICATE KEY UPDATE + name = VALUES(name), + source = VALUES(source), + browser = VALUES(browser) +` + + const loadSoftwareTitles = ` +SELECT + id +FROM + software_titles +WHERE (name, source, browser) IN (%s) +` + const deleteAllInstallersInTeam = ` +DELETE FROM + software_installers +WHERE + global_or_team_id = ? +` + + const deleteInstallersNotInList = ` +DELETE FROM + software_installers +WHERE + global_or_team_id = ? AND + title_id NOT IN (?) +` + + const insertNewOrEditedInstaller = ` +INSERT INTO software_installers ( + team_id, + global_or_team_id, + storage_id, + filename, + version, + install_script_content_id, + pre_install_query, + post_install_script_content_id, + title_id +) VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, + (SELECT id FROM software_titles WHERE name = ? AND source = ? AND browser = '') +) +ON DUPLICATE KEY UPDATE + install_script_content_id = VALUES(install_script_content_id), + post_install_script_content_id = VALUES(post_install_script_content_id), + storage_id = VALUES(storage_id), + filename = VALUES(filename), + version = VALUES(version), + pre_install_query = VALUES(pre_install_query) +` + + // use a team id of 0 if no-team + var globalOrTeamID uint + if tmID != nil { + globalOrTeamID = *tmID + } + + return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { + // if no installers are provided, just delete whatever was in + // the table + if len(installers) == 0 { + _, err := tx.ExecContext(ctx, deleteAllInstallersInTeam, globalOrTeamID) + return ctxerr.Wrap(ctx, err, "delete obsolete software installers") + } + + var args []any + for _, installer := range installers { + args = append(args, installer.Title, installer.Source, "") + } + + values := strings.TrimSuffix( + strings.Repeat("(?,?,?),", len(installers)), + ",", + ) + if _, err := tx.ExecContext(ctx, fmt.Sprintf(upsertSoftwareTitles, values), args...); err != nil { + return ctxerr.Wrap(ctx, err, "insert new/edited software title") + } + + var titleIDs []uint + if err := sqlx.SelectContext(ctx, tx, &titleIDs, fmt.Sprintf(loadSoftwareTitles, values), args...); err != nil { + return ctxerr.Wrap(ctx, err, "load existing titles") + } + + stmt, args, err := sqlx.In(deleteInstallersNotInList, globalOrTeamID, titleIDs) + if err != nil { + return ctxerr.Wrap(ctx, err, "build statement to delete obsolete installers") + } + if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { + return ctxerr.Wrap(ctx, err, "delete obsolete software installers") + } + + for _, installer := range installers { + isRes, err := insertScriptContents(ctx, installer.InstallScript, tx) + if err != nil { + return ctxerr.Wrapf(ctx, err, "inserting install script contents for software installer with name %q", installer.Filename) + } + installScriptID, _ := isRes.LastInsertId() + + var postInstallScriptID *int64 + if installer.PostInstallScript != "" { + pisRes, err := insertScriptContents(ctx, installer.PostInstallScript, tx) + if err != nil { + return ctxerr.Wrapf(ctx, err, "inserting post-install script contents for software installer with name %q", installer.Filename) + } + + insertID, _ := pisRes.LastInsertId() + postInstallScriptID = &insertID + } + + args := []interface{}{ + tmID, + globalOrTeamID, + installer.StorageID, + installer.Filename, + installer.Version, + installScriptID, + installer.PreInstallQuery, + postInstallScriptID, + installer.Title, + installer.Source, + } + + if _, err := tx.ExecContext(ctx, insertNewOrEditedInstaller, args...); err != nil { + return ctxerr.Wrapf(ctx, err, "insert new/edited installer with name %q", installer.Filename) + } + } + return nil + }) +} diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go index 62cd225a84..360ccfe98b 100644 --- a/server/datastore/mysql/software_installers_test.go +++ b/server/datastore/mysql/software_installers_test.go @@ -27,9 +27,10 @@ func TestSoftwareInstallers(t *testing.T) { fn func(t *testing.T, ds *Datastore) }{ {"SoftwareInstallRequests", testSoftwareInstallRequests}, - {"SoftwareInstallerDetails", testListSoftwareInstallerDetails}, + {"ListPendingSoftwareInstalls", testListPendingSoftwareInstalls}, {"GetSoftwareInstallResults", testGetSoftwareInstallResult}, {"CleanupUnusedSoftwareInstallers", testCleanupUnusedSoftwareInstallers}, + {"BatchSetSoftwareInstallers", testBatchSetSoftwareInstallers}, {"GetSoftwareInstallerMetadataByTeamAndTitleID", testGetSoftwareInstallerMetadataByTeamAndTitleID}, } @@ -41,7 +42,7 @@ func TestSoftwareInstallers(t *testing.T) { } } -func testListSoftwareInstallerDetails(t *testing.T, ds *Datastore) { +func testListPendingSoftwareInstalls(t *testing.T, ds *Datastore) { ctx := context.Background() host1 := test.NewHost(t, ds, "host1", "1", "host1key", "host1uuid", time.Now()) @@ -415,6 +416,115 @@ func testCleanupUnusedSoftwareInstallers(t *testing.T, ds *Datastore) { assertExisting(nil) } +func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { + ctx := context.Background() + + // create a team + team, err := ds.NewTeam(ctx, &fleet.Team{Name: t.Name()}) + require.NoError(t, err) + + // TODO(roberto): perform better assertions, we should have evertything + // to check that the actual values of everything match. + assertSoftware := func(wantTitles []fleet.SoftwareTitle) { + tmFilter := fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}} + titles, _, _, err := ds.ListSoftwareTitles( + ctx, + fleet.SoftwareTitleListOptions{TeamID: &team.ID}, + tmFilter, + ) + require.NoError(t, err) + require.Len(t, titles, len(wantTitles)) + + for _, title := range titles { + meta, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, &team.ID, title.ID) + require.NoError(t, err) + require.NotNil(t, meta.TitleID) + } + } + + // batch set with everything empty + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, nil) + require.NoError(t, err) + assertSoftware(nil) + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{}) + require.NoError(t, err) + assertSoftware(nil) + + // add a single installer + ins0 := "installer0" + ins0File := bytes.NewReader([]byte("installer0")) + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{{ + InstallScript: "install", + InstallerFile: ins0File, + StorageID: ins0, + Filename: "installer0", + Title: "ins0", + Source: "apps", + Version: "1", + PreInstallQuery: "foo", + }}) + require.NoError(t, err) + assertSoftware([]fleet.SoftwareTitle{ + {Name: ins0, Source: "apps", Browser: ""}, + }) + + // add a new installer + ins0 installer + ins1 := "installer1" + ins1File := bytes.NewReader([]byte("installer1")) + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{ + { + InstallScript: "install", + InstallerFile: ins0File, + StorageID: ins0, + Filename: ins0, + Title: ins0, + Source: "apps", + Version: "1", + PreInstallQuery: "select 0 from foo;", + }, + { + InstallScript: "install", + PostInstallScript: "post-install", + InstallerFile: ins1File, + StorageID: ins1, + Filename: ins1, + Title: ins1, + Source: "apps", + Version: "2", + PreInstallQuery: "select 1 from bar;", + }, + }) + require.NoError(t, err) + assertSoftware([]fleet.SoftwareTitle{ + {Name: ins0, Source: "apps", Browser: ""}, + {Name: ins1, Source: "apps", Browser: ""}, + }) + + // remove ins0 + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{ + { + InstallScript: "install", + PostInstallScript: "post-install", + InstallerFile: ins1File, + StorageID: ins1, + Filename: ins1, + Title: ins1, + Source: "apps", + Version: "2", + PreInstallQuery: "select 1 from bar;", + }, + }) + require.NoError(t, err) + assertSoftware([]fleet.SoftwareTitle{ + {Name: ins1, Source: "apps", Browser: ""}, + }) + + // remove everything + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{}) + require.NoError(t, err) + assertSoftware([]fleet.SoftwareTitle{}) +} + func testGetSoftwareInstallerMetadataByTeamAndTitleID(t *testing.T, ds *Datastore) { ctx := context.Background() team, err := ds.NewTeam(ctx, &fleet.Team{Name: "team 2"}) diff --git a/server/datastore/mysql/software_titles.go b/server/datastore/mysql/software_titles.go index 273cbed594..edf3066b67 100644 --- a/server/datastore/mysql/software_titles.go +++ b/server/datastore/mysql/software_titles.go @@ -193,8 +193,6 @@ func spliceSecondaryOrderBySoftwareTitlesSQL(stmt string, opts fleet.ListOptions return strings.Replace(stmt, targetSubstr, targetSubstr+secondaryOrderBy, 1) } -// TODO: Does this need to be updated to include software installers? Otherwise, this list won't -// include software packages that haven't been installed on any hosts (see SoftwareTitleByID above). func selectSoftwareTitlesSQL(opt fleet.SoftwareTitleListOptions) (string, []any) { stmt := ` SELECT diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index ec626b157a..71d7e2605f 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -493,7 +493,8 @@ type Datastore interface { // InsertSoftwareInstallRequest tracks a new request to install the provided software installer in the host InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareTitleID uint) error - // GetSoftwareInstallerForTitle TODO + // GetSoftwareInstallerForTitle returns the software installer + // associated with the given title - team combination. GetSoftwareInstallerForTitle(ctx context.Context, softwareTitleID uint, teamID *uint) (*SoftwareInstaller, error) /////////////////////////////////////////////////////////////////////////////// @@ -1503,6 +1504,9 @@ type Datastore interface { // CleanupUnusedSoftwareInstallers will remove software installers that have // no references to them from the software_installers table. CleanupUnusedSoftwareInstallers(ctx context.Context, softwareInstallStore SoftwareInstallerStore) error + + // BatchSetSoftwareInstallers sets the software installers for the given team or no team. + BatchSetSoftwareInstallers(ctx context.Context, tmID *uint, installers []*UploadSoftwareInstallerPayload) error } // MDMAppleStore wraps nanomdm's storage and adds methods to deal with diff --git a/server/fleet/scripts.go b/server/fleet/scripts.go index 3bffbcef32..70ccb510d3 100644 --- a/server/fleet/scripts.go +++ b/server/fleet/scripts.go @@ -358,6 +358,13 @@ type ScriptPayload struct { ScriptContents []byte `json:"script_contents"` } +type SoftwareInstallerPayload struct { + URL string `json:"url"` + PreInstallQuery string `json:"pre_install_query"` + InstallScript string `json:"install_script"` + PostInstallScript string `json:"post_install_script"` +} + type HostLockWipeStatus struct { // HostFleetPlatform is the fleet-normalized platform of the host, i.e. the // result of host.FleetPlatform(). diff --git a/server/fleet/service.go b/server/fleet/service.go index 50f3922532..f657d96a8c 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -638,6 +638,10 @@ type Service interface { // GetSoftwareInstallResults gets the results for a particular software install attempt. GetSoftwareInstallResults(ctx context.Context, installUUID string) (*HostSoftwareInstallerResult, error) + // BatchSetSoftwareInstallers replaces the software installers for a + // specified team + BatchSetSoftwareInstallers(ctx context.Context, tmName string, payloads []SoftwareInstallerPayload, dryRun bool) error + // ///////////////////////////////////////////////////////////////////////////// // Vulnerabilities diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index e91909ae4b..52f083ce0e 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "path/filepath" "time" ) @@ -200,16 +199,16 @@ type DownloadSoftwareInstallerPayload struct { Size int64 } -func SofwareInstallerSourceFromFilename(filename string) (string, error) { - switch ext := filepath.Ext(filename); ext { - case ".deb": +func SofwareInstallerSourceFromExtension(ext string) (string, error) { + switch ext { + case "deb": return "deb_packages", nil - case ".exe", ".msi": + case "exe", "msi": return "programs", nil - case ".pkg": + case "pkg": return "pkg_packages", nil default: - return "", fmt.Errorf("unsupported file type: %s", filename) + return "", fmt.Errorf("unsupported file type: %s", ext) } } diff --git a/server/fleet/teams.go b/server/fleet/teams.go index db83210230..d7c41f1d8b 100644 --- a/server/fleet/teams.go +++ b/server/fleet/teams.go @@ -141,13 +141,14 @@ func (t *Team) UnmarshalJSON(b []byte) error { type TeamConfig struct { // AgentOptions is the options for osquery and Orbit. - AgentOptions *json.RawMessage `json:"agent_options,omitempty"` - HostExpirySettings HostExpirySettings `json:"host_expiry_settings"` - WebhookSettings TeamWebhookSettings `json:"webhook_settings"` - Integrations TeamIntegrations `json:"integrations"` - Features Features `json:"features"` - MDM TeamMDM `json:"mdm"` - Scripts optjson.Slice[string] `json:"scripts,omitempty"` + AgentOptions *json.RawMessage `json:"agent_options,omitempty"` + HostExpirySettings HostExpirySettings `json:"host_expiry_settings"` + WebhookSettings TeamWebhookSettings `json:"webhook_settings"` + Integrations TeamIntegrations `json:"integrations"` + Features Features `json:"features"` + MDM TeamMDM `json:"mdm"` + Scripts optjson.Slice[string] `json:"scripts,omitempty"` + Software optjson.Slice[TeamSpecSoftware] `json:"software,omitempty"` } type TeamWebhookSettings struct { @@ -156,6 +157,17 @@ type TeamWebhookSettings struct { FailingPoliciesWebhook FailingPoliciesWebhookSettings `json:"failing_policies_webhook"` } +type TeamSpecSoftwareAsset struct { + Path string `json:"path"` +} + +type TeamSpecSoftware struct { + URL string `json:"url"` + PreInstallQuery TeamSpecSoftwareAsset `json:"pre_install_query"` + InstallScript TeamSpecSoftwareAsset `json:"install_script"` + PostInstallScript TeamSpecSoftwareAsset `json:"post_install_script"` +} + type TeamMDM struct { EnableDiskEncryption bool `json:"enable_disk_encryption"` MacOSUpdates MacOSUpdates `json:"macos_updates"` @@ -404,14 +416,15 @@ type TeamSpec struct { // If the agent_options key is present but empty in the YAML, will be set to // "null" (JSON null). Otherwise, if the key is present and set, it will be // set to the agent options JSON object. - AgentOptions json.RawMessage `json:"agent_options,omitempty"` // marshals as "null" if omitempty is not set - HostExpirySettings *HostExpirySettings `json:"host_expiry_settings,omitempty"` - Secrets []EnrollSecret `json:"secrets,omitempty"` - Features *json.RawMessage `json:"features"` - MDM TeamSpecMDM `json:"mdm"` - Scripts optjson.Slice[string] `json:"scripts"` - WebhookSettings TeamSpecWebhookSettings `json:"webhook_settings"` - Integrations TeamSpecIntegrations `json:"integrations"` + AgentOptions json.RawMessage `json:"agent_options,omitempty"` // marshals as "null" if omitempty is not set + HostExpirySettings *HostExpirySettings `json:"host_expiry_settings,omitempty"` + Secrets []EnrollSecret `json:"secrets,omitempty"` + Features *json.RawMessage `json:"features"` + MDM TeamSpecMDM `json:"mdm"` + Scripts optjson.Slice[string] `json:"scripts"` + WebhookSettings TeamSpecWebhookSettings `json:"webhook_settings"` + Integrations TeamSpecIntegrations `json:"integrations"` + Software optjson.Slice[TeamSpecSoftware] `json:"software,omitempty"` } type TeamSpecWebhookSettings struct { diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index a18a83fc85..923e6035f0 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -951,6 +951,8 @@ type GetSoftwareInstallResultsFunc func(ctx context.Context, resultsUUID string) type CleanupUnusedSoftwareInstallersFunc func(ctx context.Context, softwareInstallStore fleet.SoftwareInstallerStore) error +type BatchSetSoftwareInstallersFunc func(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error + type DataStore struct { HealthCheckFunc HealthCheckFunc HealthCheckFuncInvoked bool @@ -2350,6 +2352,9 @@ type DataStore struct { CleanupUnusedSoftwareInstallersFunc CleanupUnusedSoftwareInstallersFunc CleanupUnusedSoftwareInstallersFuncInvoked bool + BatchSetSoftwareInstallersFunc BatchSetSoftwareInstallersFunc + BatchSetSoftwareInstallersFuncInvoked bool + mu sync.Mutex } @@ -5614,3 +5619,10 @@ func (s *DataStore) CleanupUnusedSoftwareInstallers(ctx context.Context, softwar s.mu.Unlock() return s.CleanupUnusedSoftwareInstallersFunc(ctx, softwareInstallStore) } + +func (s *DataStore) BatchSetSoftwareInstallers(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + s.mu.Lock() + s.BatchSetSoftwareInstallersFuncInvoked = true + s.mu.Unlock() + return s.BatchSetSoftwareInstallersFunc(ctx, tmID, installers) +} diff --git a/server/service/client.go b/server/service/client.go index 53d8cc2fae..4d6fe83d4a 100644 --- a/server/service/client.go +++ b/server/service/client.go @@ -559,6 +559,65 @@ func (c *Client) ApplyGroup( tmScriptsPayloads[k] = scriptPayloads } + tmSoftware := extractTmSpecsSoftware(specs.Teams) + tmSoftwarePayloads := make(map[string][]fleet.SoftwareInstallerPayload, len(tmScripts)) + for tmName, software := range tmSoftware { + softwarePayloads := make([]fleet.SoftwareInstallerPayload, len(software)) + for i, si := range software { + var qc string + var err error + if si.PreInstallQuery.Path != "" { + queryFile := resolveApplyRelativePath(baseDir, si.PreInstallQuery.Path) + rawSpec, err := os.ReadFile(queryFile) + if err != nil { + return nil, fmt.Errorf("reading pre-install query: %w", err) + } + + group, err := spec.GroupFromBytes(rawSpec) + if err != nil { + return nil, fmt.Errorf("unable to parse query spec file %s: %w", queryFile, err) + } + + if len(group.Queries) > 1 { + return nil, fmt.Errorf("pre_install_query file %s contains more than one query", queryFile) + } + + if len(group.Queries) == 0 { + return nil, fmt.Errorf("pre_install_query file %s doesn't have a query defined", queryFile) + } + + qc = group.Queries[0].Query + } + + var ic []byte + if si.InstallScript.Path != "" { + installScriptFile := resolveApplyRelativePath(baseDir, si.InstallScript.Path) + ic, err = os.ReadFile(installScriptFile) + if err != nil { + return nil, fmt.Errorf("applying fleet config: %w", err) + } + } + + var pc []byte + if si.PostInstallScript.Path != "" { + postInstallScriptFile := resolveApplyRelativePath(baseDir, si.PostInstallScript.Path) + pc, err = os.ReadFile(postInstallScriptFile) + if err != nil { + return nil, fmt.Errorf("applying fleet config: %w", err) + } + } + + softwarePayloads[i] = fleet.SoftwareInstallerPayload{ + URL: si.URL, + PreInstallQuery: qc, + InstallScript: string(ic), + PostInstallScript: string(pc), + } + } + + tmSoftwarePayloads[tmName] = softwarePayloads + } + // Next, apply the teams specs before saving the profiles, so that any // non-existing team gets created. var err error @@ -605,6 +664,13 @@ func (c *Client) ApplyGroup( } } } + if len(tmSoftwarePayloads) > 0 { + for tmName, software := range tmSoftwarePayloads { + if err := c.ApplyTeamSoftwareInstallers(tmName, software, opts); err != nil { + return nil, fmt.Errorf("applying software installers for team %q: %w", tmName, err) + } + } + } if opts.DryRun { logfn("[+] would've applied %d teams\n", len(specs.Teams)) } else { @@ -828,6 +894,38 @@ func extractTmSpecsMDMCustomSettings(tmSpecs []json.RawMessage) map[string][]fle return m } +func extractTmSpecsSoftware(tmSpecs []json.RawMessage) map[string][]fleet.TeamSpecSoftware { + var m map[string][]fleet.TeamSpecSoftware + for _, tm := range tmSpecs { + var spec struct { + Name string `json:"name"` + Software json.RawMessage `json:"software"` + } + if err := json.Unmarshal(tm, &spec); err != nil { + // ignore, this will fail in the call to apply team specs + continue + } + spec.Name = norm.NFC.String(spec.Name) + if spec.Name != "" && len(spec.Software) > 0 { + if m == nil { + m = make(map[string][]fleet.TeamSpecSoftware) + } + var software []fleet.TeamSpecSoftware + if err := json.Unmarshal(spec.Software, &software); err != nil { + // ignore, will fail in apply team specs call + continue + } + if software == nil { + // to be consistent with the AppConfig custom settings, set it to an + // empty slice if the provided custom settings are present but empty. + software = []fleet.TeamSpecSoftware{} + } + m[spec.Name] = software + } + } + return m +} + func extractTmSpecsScripts(tmSpecs []json.RawMessage) map[string][]string { var m map[string][]string for _, tm := range tmSpecs { diff --git a/server/service/client_mdm.go b/server/service/client_mdm.go index f4b52c24d0..4eb82d0968 100644 --- a/server/service/client_mdm.go +++ b/server/service/client_mdm.go @@ -14,7 +14,6 @@ import ( "net/http" "net/url" "os" - "path" "path/filepath" "strings" @@ -158,30 +157,6 @@ func (c *Client) ValidateBootstrapPackageFromURL(url string) (*fleet.MDMAppleBoo return downloadRemoteMacosBootstrapPackage(url) } -func extractFilenameFromPath(p string) string { - u, err := url.Parse(p) - if err != nil { - return "" - } - - invalid := map[string]struct{}{ - "": {}, - ".": {}, - "/": {}, - } - - b := path.Base(u.Path) - if _, ok := invalid[b]; ok { - return "" - } - - if _, ok := invalid[path.Ext(b)]; ok { - return b + ".pkg" - } - - return b -} - func downloadRemoteMacosBootstrapPackage(pkgURL string) (*fleet.MDMAppleBootstrapPackage, error) { resp, err := http.Get(pkgURL) // nolint:gosec // we want this URL to be provided by the user. It will run on their machine. if err != nil { @@ -205,7 +180,7 @@ func downloadRemoteMacosBootstrapPackage(pkgURL string) (*fleet.MDMAppleBootstra // if it fails, try to extract it from the URL if filename == "" { - filename = extractFilenameFromPath(pkgURL) + filename = file.ExtractFilenameFromURLPath(pkgURL, "pkg") } // if all else fails, use a default name diff --git a/server/service/client_teams.go b/server/service/client_teams.go index bfa8f4b5d4..8acbd0998e 100644 --- a/server/service/client_teams.go +++ b/server/service/client_teams.go @@ -92,3 +92,13 @@ func (c *Client) ApplyTeamScripts(tmName string, scripts []fleet.ScriptPayload, query.Add("team_name", tmName) return c.authenticatedRequestWithQuery(map[string]interface{}{"scripts": scripts}, verb, path, nil, query.Encode()) } + +func (c *Client) ApplyTeamSoftwareInstallers(tmName string, softwareInstallers []fleet.SoftwareInstallerPayload, opts fleet.ApplySpecOptions) error { + verb, path := "POST", "/api/latest/fleet/software/batch" + query, err := url.ParseQuery(opts.RawQuery()) + if err != nil { + return err + } + query.Add("team_name", tmName) + return c.authenticatedRequestWithQuery(map[string]interface{}{"software": softwareInstallers}, verb, path, nil, query.Encode()) +} diff --git a/server/service/client_test.go b/server/service/client_test.go index 8814f2e3f6..c33ac3ecc7 100644 --- a/server/service/client_test.go +++ b/server/service/client_test.go @@ -462,29 +462,6 @@ spec: } } -func TestExtractFilenameFromPath(t *testing.T) { - cases := []struct { - in string - out string - }{ - {"http://example.com", ""}, - {"http://example.com/", ""}, - {"http://example.com?foo=bar", ""}, - {"http://example.com/foo.pkg", "foo.pkg"}, - {"http://example.com/foo.exe", "foo.exe"}, - {"http://example.com/foo.pkg?bar=baz", "foo.pkg"}, - {"http://example.com/foo.bar.pkg", "foo.bar.pkg"}, - {"http://example.com/foo", "foo.pkg"}, - {"http://example.com/foo/bar/baz", "baz.pkg"}, - {"http://example.com/foo?bar=baz", "foo.pkg"}, - } - - for _, c := range cases { - got := extractFilenameFromPath(c.in) - require.Equalf(t, c.out, got, "for URL %s", c.in) - } -} - func TestGetProfilesContents(t *testing.T) { tempDir := t.TempDir() darwinProfile := mobileconfigForTest("bar", "I") diff --git a/server/service/handler.go b/server/service/handler.go index 4d0bae48ab..c6fba1ef31 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -377,6 +377,7 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC ue.POST("/api/_version_/fleet/software/package", uploadSoftwareInstallerEndpoint, uploadSoftwareInstallerRequest{}) ue.DELETE("/api/_version_/fleet/software/{title_id:[0-9]+}/package", deleteSoftwareInstallerEndpoint, deleteSoftwareInstallerRequest{}) ue.GET("/api/_version_/fleet/software/install/results/{install_uuid}", getSoftwareInstallResultsEndpoint, getSoftwareInstallResultsRequest{}) + ue.POST("/api/_version_/fleet/software/batch", batchSetSoftwareInstallersEndpoint, batchSetSoftwareInstallersRequest{}) // Vulnerabilities ue.GET("/api/_version_/fleet/vulnerabilities", listVulnerabilitiesEndpoint, listVulnerabilitiesRequest{}) diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 0e5e8fd974..b92d7bfd4b 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -8975,6 +8975,200 @@ func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerUploadDownloadAndD }) } +func (s *integrationEnterpriseTestSuite) TestApplyTeamsSoftwareConfig() { + t := s.T() + + // create a team through the service so it initializes the agent ops + teamName := t.Name() + "team1" + team := &fleet.Team{ + Name: teamName, + Description: "desc team1", + } + var createTeamResp teamResponse + s.DoJSON("POST", "/api/latest/fleet/teams", team, http.StatusOK, &createTeamResp) + require.NotZero(t, createTeamResp.Team.ID) + team = createTeamResp.Team + + // apply with software + // must not use applyTeamSpecsRequest and marshal it as JSON, as it will set + // all keys to their zerovalue, and some are only valid with mdm enabled. + teamSpecs := map[string]any{ + "specs": []any{ + map[string]any{ + "name": teamName, + "software": []map[string]any{ + { + "url": "http://foo.com", + "install_script": map[string]string{ + "path": "./foo/install-script.sh", + }, + "post_install_script": map[string]string{ + "path": "./foo/post-install-script.sh", + }, + "pre_install_query": map[string]string{ + "path": "./foo/query.yaml", + }, + }, + { + "url": "http://bar.com", + "install_script": map[string]string{ + "path": "./bar/install-script.sh", + }, + "post_install_script": map[string]string{ + "path": "./bar/post-install-script.sh", + }, + "pre_install_query": map[string]string{ + "path": "./bar/query.yaml", + }, + }, + }, + }, + }, + } + s.Do("POST", "/api/latest/fleet/spec/teams", teamSpecs, http.StatusOK) + + wantSoftware := []fleet.TeamSpecSoftware{ + { + URL: "http://foo.com", + InstallScript: fleet.TeamSpecSoftwareAsset{Path: "./foo/install-script.sh"}, + PostInstallScript: fleet.TeamSpecSoftwareAsset{Path: "./foo/post-install-script.sh"}, + PreInstallQuery: fleet.TeamSpecSoftwareAsset{Path: "./foo/query.yaml"}, + }, + { + URL: "http://bar.com", + InstallScript: fleet.TeamSpecSoftwareAsset{Path: "./bar/install-script.sh"}, + PostInstallScript: fleet.TeamSpecSoftwareAsset{Path: "./bar/post-install-script.sh"}, + PreInstallQuery: fleet.TeamSpecSoftwareAsset{Path: "./bar/query.yaml"}, + }, + } + + // retrieving the team returns the software + var teamResp getTeamResponse + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/teams/%d", team.ID), nil, http.StatusOK, &teamResp) + require.Equal(t, wantSoftware, teamResp.Team.Config.Software.Value) + + // apply without custom software specified, should not replace existing software + teamSpecs = map[string]any{ + "specs": []any{ + map[string]any{ + "name": teamName, + }, + }, + } + s.Do("POST", "/api/latest/fleet/spec/teams", teamSpecs, http.StatusOK) + teamResp = getTeamResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/teams/%d", team.ID), nil, http.StatusOK, &teamResp) + require.Equal(t, wantSoftware, teamResp.Team.Config.Software.Value) + + // apply with explicitly empty custom software would clear the existing + // software, but dry-run + teamSpecs = map[string]any{ + "specs": []any{ + map[string]any{ + "name": teamName, + "software": nil, + }, + }, + } + s.Do("POST", "/api/latest/fleet/spec/teams", teamSpecs, http.StatusOK, "dry_run", "true") + teamResp = getTeamResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/teams/%d", team.ID), nil, http.StatusOK, &teamResp) + require.Equal(t, wantSoftware, teamResp.Team.Config.Software.Value) + + // apply with explicitly empty software clears the existing software + teamSpecs = map[string]any{ + "specs": []any{ + map[string]any{ + "name": teamName, + "software": nil, + }, + }, + } + s.Do("POST", "/api/latest/fleet/spec/teams", teamSpecs, http.StatusOK) + teamResp = getTeamResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/teams/%d", team.ID), nil, http.StatusOK, &teamResp) + require.Empty(t, teamResp.Team.Config.Software.Value) + + // patch with an invalid array returns an error + teamSpecs = map[string]any{ + "specs": []any{ + map[string]any{ + "name": teamName, + "software": []any{"foo", 1}, + }, + }, + } + s.Do("POST", "/api/latest/fleet/spec/teams", teamSpecs, http.StatusBadRequest) + teamResp = getTeamResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/teams/%d", team.ID), nil, http.StatusOK, &teamResp) + require.Empty(t, teamResp.Team.Config.Software.Value) +} + +func (s *integrationMDMTestSuite) TestBatchSetSoftwareInstallers() { + t := s.T() + + // a team name is required (we don't allow installers for "no team") + s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{}, http.StatusBadRequest) + + // non-existent team + s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{}, http.StatusNotFound, "team_name", "foo") + + // create a team + tm, err := s.ds.NewTeam(context.Background(), &fleet.Team{ + Name: t.Name(), + Description: "desc", + }) + require.NoError(t, err) + + // software with a bad URL + softwareToInstall := []fleet.SoftwareInstallerPayload{ + {URL: "."}, + } + s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusUnprocessableEntity, "team_name", tm.Name) + + // create an HTTP server to host the software installer + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + file, err := os.Open(filepath.Join("testdata", "software-installers", "ruby.deb")) + require.NoError(t, err) + defer file.Close() + w.Header().Set("Content-Type", "application/vnd.debian.binary-package") + _, err = io.Copy(w, file) + require.NoError(t, err) + }) + + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + + // do a request with a valid URL + softwareToInstall = []fleet.SoftwareInstallerPayload{ + {URL: srv.URL}, + } + s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusNoContent, "team_name", tm.Name) + + // TODO(roberto): test with a variety of response codes + + // check the application status + titlesResp := listSoftwareTitlesResponse{} + s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &titlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(tm.ID))) + require.Equal(t, 1, titlesResp.Count) + require.Len(t, titlesResp.SoftwareTitles, 1) + + // same payload doesn't modify anything + s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusNoContent, "team_name", tm.Name) + newTitlesResp := listSoftwareTitlesResponse{} + s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &newTitlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(tm.ID))) + require.Equal(t, titlesResp, newTitlesResp) + + // empty payload cleans the software items + softwareToInstall = []fleet.SoftwareInstallerPayload{} + s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusNoContent, "team_name", tm.Name) + titlesResp = listSoftwareTitlesResponse{} + s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &titlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(tm.ID))) + require.Equal(t, 0, titlesResp.Count) + require.Len(t, titlesResp.SoftwareTitles, 0) + +} + func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerNewInstallRequestPlatformValidation() { t := s.T() diff --git a/server/service/software_installers.go b/server/service/software_installers.go index d2528401fe..4530d97c62 100644 --- a/server/service/software_installers.go +++ b/server/service/software_installers.go @@ -278,3 +278,37 @@ func (svc *Service) GetSoftwareInstallResults(ctx context.Context, resultUUID st return nil, fleet.ErrMissingLicense } + +//////////////////////////////////////////////////////////////////////////////// +// Batch replace software installers +//////////////////////////////////////////////////////////////////////////////// + +type batchSetSoftwareInstallersRequest struct { + TeamName string `json:"-" query:"team_name"` + DryRun bool `json:"-" query:"dry_run,optional"` // if true, apply validation but do not save changes + Software []fleet.SoftwareInstallerPayload `json:"software"` +} + +type batchSetSoftwareInstallersResponse struct { + Err error `json:"error,omitempty"` +} + +func (r batchSetSoftwareInstallersResponse) error() error { return r.Err } + +func (r batchSetSoftwareInstallersResponse) Status() int { return http.StatusNoContent } + +func batchSetSoftwareInstallersEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { + req := request.(*batchSetSoftwareInstallersRequest) + if err := svc.BatchSetSoftwareInstallers(ctx, req.TeamName, req.Software, req.DryRun); err != nil { + return batchSetSoftwareInstallersResponse{Err: err}, nil + } + return batchSetSoftwareInstallersResponse{}, nil +} + +func (svc *Service) BatchSetSoftwareInstallers(ctx context.Context, tmName string, payloads []fleet.SoftwareInstallerPayload, dryRun bool) error { + // skipauth: No authorization check needed due to implementation returning + // only license error. + svc.authz.SkipAuthorization(ctx) + + return fleet.ErrMissingLicense +} From 5a50e5d4b2d057816a98e0825e18dc0b355ab15b Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Tue, 14 May 2024 13:10:12 -0500 Subject: [PATCH 40/56] Fix unreleased issues in software installers UI: Part 4 (#18987) --- frontend/components/Editor/Editor.tsx | 2 +- .../DataTable/TextCell/TextCell.tsx | 2 +- .../AddSoftwareForm/AddSoftwareForm.tsx | 9 +- .../SoftwarePage/components/icons/Falcon.tsx | 24 +++++ .../SoftwarePage/components/icons/index.ts | 4 +- .../Software/HostSoftwareTableConfig.tsx | 94 +++++++++++++------ .../hosts/details/cards/Software/Software.tsx | 10 +- .../hosts/details/cards/Software/_styles.scss | 72 ++++++++++++-- 8 files changed, 170 insertions(+), 47 deletions(-) create mode 100644 frontend/pages/SoftwarePage/components/icons/Falcon.tsx diff --git a/frontend/components/Editor/Editor.tsx b/frontend/components/Editor/Editor.tsx index d5c39d3d01..c9bd360508 100644 --- a/frontend/components/Editor/Editor.tsx +++ b/frontend/components/Editor/Editor.tsx @@ -8,7 +8,7 @@ const baseClass = "editor"; interface IEditorProps { focus?: boolean; label?: string; - labelTooltip?: string; + labelTooltip?: string | JSX.Element; error?: string | null; readOnly?: boolean; /** diff --git a/frontend/components/TableContainer/DataTable/TextCell/TextCell.tsx b/frontend/components/TableContainer/DataTable/TextCell/TextCell.tsx index bdf6f2137a..a699438c85 100644 --- a/frontend/components/TableContainer/DataTable/TextCell/TextCell.tsx +++ b/frontend/components/TableContainer/DataTable/TextCell/TextCell.tsx @@ -51,7 +51,7 @@ const TextCell = ({ }; return ( - + {formatter(val) || renderEmptyCell()} ); diff --git a/frontend/pages/SoftwarePage/components/AddSoftwareForm/AddSoftwareForm.tsx b/frontend/pages/SoftwarePage/components/AddSoftwareForm/AddSoftwareForm.tsx index 4f7fee9dbc..6f7fdfc001 100644 --- a/frontend/pages/SoftwarePage/components/AddSoftwareForm/AddSoftwareForm.tsx +++ b/frontend/pages/SoftwarePage/components/AddSoftwareForm/AddSoftwareForm.tsx @@ -184,7 +184,14 @@ const AddSoftwareForm = ({ value={formData.installScript} helpText="Fleet will run this command on hosts to install software." label="Install script" - labelTooltip="For security agents, add the script provided by the vendor." + labelTooltip={ + <> + For security agents, add the script provided by the vendor. +
+ In custom scripts, you can use the $INSTALLER_PATH variable to + point to the installer. + + } /> )} ) => ( + + + + +); + +export default Falcon; diff --git a/frontend/pages/SoftwarePage/components/icons/index.ts b/frontend/pages/SoftwarePage/components/icons/index.ts index 8ac710679f..82a9b00f40 100644 --- a/frontend/pages/SoftwarePage/components/icons/index.ts +++ b/frontend/pages/SoftwarePage/components/icons/index.ts @@ -17,7 +17,7 @@ import Word from "./Word"; import Zoom from "./Zoom"; import ChromeOS from "./ChromeOS"; import LinuxOS from "./LinuxOS"; -// import Falcon from "./Falcon"; // TODO: Add Falcon icon svg +import Falcon from "./Falcon"; // SOFTWARE_NAME_TO_ICON_MAP list "special" applications that have a defined // icon for them, keys refer to application names, and are intended to be fuzzy @@ -26,6 +26,7 @@ export const SOFTWARE_NAME_TO_ICON_MAP = { "adobe acrobat reader": AcrobatReader, "google chrome": ChromeApp, "microsoft excel": Excel, + falcon: Falcon, firefox: Firefox, package: Package, safari: Safari, @@ -34,7 +35,6 @@ export const SOFTWARE_NAME_TO_ICON_MAP = { "visual studio code": VisualStudioCode, "microsoft word": Word, zoom: Zoom, - // falcon: Falcon, darwin: MacOS, windows: WindowsOS, chrome: ChromeOS, diff --git a/frontend/pages/hosts/details/cards/Software/HostSoftwareTableConfig.tsx b/frontend/pages/hosts/details/cards/Software/HostSoftwareTableConfig.tsx index 60d27f7f4d..9b73075d9d 100644 --- a/frontend/pages/hosts/details/cards/Software/HostSoftwareTableConfig.tsx +++ b/frontend/pages/hosts/details/cards/Software/HostSoftwareTableConfig.tsx @@ -47,49 +47,73 @@ type IInstalledVersionsCellProps = CellProps< type IVulnerabilitiesCellProps = IInstalledVersionsCellProps; // type IActionsCellProps = CellProps; -const generateActions = ( - softwareId: number, - status: SoftwareInstallStatus | null, - installingSoftwareId: number | null, - canInstall: boolean, - packageToInstall?: string | null -) => { +const generateActions = ({ + canInstall, + installingSoftwareId, + isFleetdHost, + softwareId, + status, + packageToInstall, +}: { + canInstall: boolean; + installingSoftwareId: number | null; + isFleetdHost: boolean; + softwareId: number; + status: SoftwareInstallStatus | null; + packageToInstall?: string | null; +}) => { // this gives us a clean slate of the default actions so we can modify // the options. - let actions = cloneDeep(DEFAULT_ACTION_OPTIONS); + const actions = cloneDeep(DEFAULT_ACTION_OPTIONS); + + const indexInstallAction = actions.findIndex((a) => a.value === "install"); + if (indexInstallAction === -1) { + // this should never happen unless the default actions change, but if it does we'll throw an + // error to fail loudly so that we know to update this function + throw new Error("Install action not found in default actions"); + } // remove install if there is no package to install if (!packageToInstall || !canInstall) { - actions = actions.filter((action) => action.value !== "install"); + actions.splice(indexInstallAction, 1); + return actions; + } + + // disable install option if not a fleetd host + if (!isFleetdHost) { + actions[indexInstallAction].disabled = true; + actions[indexInstallAction].tooltipContent = + "To install software on this host, deploy the fleetd agent with --enable-scripts and refetch host vitals."; + return actions; } // disable install option if software is already installing if (softwareId === installingSoftwareId || status === "pending") { - const installAction = actions.find((action) => action.value === "install"); - if (installAction) { - installAction.disabled = true; - } + actions[indexInstallAction].disabled = true; + return actions; } return actions; }; interface ISoftwareTableHeadersProps { - installingSoftwareId: number | null; - onSelectAction: (software: IHostSoftware, action: string) => void; canInstall: boolean; + installingSoftwareId: number | null; + isFleetdHost: boolean; router: InjectedRouter; teamId: number; + onSelectAction: (software: IHostSoftware, action: string) => void; } // NOTE: cellProps come from react-table // more info here https://react-table.tanstack.com/docs/api/useTable#cell-properties export const generateSoftwareTableHeaders = ({ - router, - installingSoftwareId, - onSelectAction, canInstall, + installingSoftwareId, + isFleetdHost, + router, teamId, + onSelectAction, }: ISoftwareTableHeadersProps): ISoftwareTableConfig[] => { const tableHeaders: ISoftwareTableConfig[] = [ { @@ -168,19 +192,27 @@ export const generateSoftwareTableHeaders = ({ // the accessor here is insignificant, we just need it as its required // but we don't use it. accessor: "id", - Cell: (cellProps: ITableNumberCellProps) => ( - onSelectAction(cellProps.row.original, action)} - /> - ), + Cell: ({ row: { original } }: ITableNumberCellProps) => { + const { + id: softwareId, + status, + package_available_for_install: packageToInstall, + } = original; + return ( + onSelectAction(original, action)} + /> + ); + }, }, ]; diff --git a/frontend/pages/hosts/details/cards/Software/Software.tsx b/frontend/pages/hosts/details/cards/Software/Software.tsx index a19c7b3acb..0ce821199e 100644 --- a/frontend/pages/hosts/details/cards/Software/Software.tsx +++ b/frontend/pages/hosts/details/cards/Software/Software.tsx @@ -152,11 +152,9 @@ const SoftwareCard = ({ [isMyDevicePage, refetchDeviceSoftware, refetchHostSoftware] ); - const canInstallSoftware = - isFleetdHost && - Boolean( - isGlobalAdmin || isGlobalMaintainer || isTeamAdmin || isTeamMaintainer - ); + const canInstallSoftware = Boolean( + isGlobalAdmin || isGlobalMaintainer || isTeamAdmin || isTeamMaintainer + ); const installHostSoftwarePackage = useCallback( async (softwareId: number) => { @@ -206,6 +204,7 @@ const SoftwareCard = ({ canInstall: canInstallSoftware, onSelectAction, teamId, + isFleetdHost, }); }, [ isMyDevicePage, @@ -214,6 +213,7 @@ const SoftwareCard = ({ canInstallSoftware, onSelectAction, teamId, + isFleetdHost, ]); const renderSoftwareTable = () => { diff --git a/frontend/pages/hosts/details/cards/Software/_styles.scss b/frontend/pages/hosts/details/cards/Software/_styles.scss index 780245ac89..9b80d40c54 100644 --- a/frontend/pages/hosts/details/cards/Software/_styles.scss +++ b/frontend/pages/hosts/details/cards/Software/_styles.scss @@ -3,10 +3,70 @@ width: 325px; // Custom to fit placeholder text } - // // TODO: Addingoverflow-x: scroll to the table clips the actions dropdown at the bottom of the - // // table. Find a solution that allows dropdown menu to be displayed over the bottom of the table - // // in the y-axis when the table is scrollable in the x-axis. - // .data-table-block .data-table__wrapper { - // overflow-x: scroll; - // } + .host-software-table { + .data-table-block { + .data-table__table { + thead { + .status__header { + display: table-cell; + } + .source__header { + display: table-cell; + } + .Vulnerabilities__header { + display: table-cell; + } + @media (max-width: $break-xl) { + .source__header { + display: none; + width: 0; + } + } + @media (max-width: $break-lg) { + .Vulnerabilities__header { + display: none; + width: 0; + } + } + @media (max-width: $break-sm) { + .status__header { + display: none; + width: 0; + } + } + } + tbody { + tr { + .status__cell { + display: table-cell; + } + .source__cell { + display: table-cell; + } + .Vulnerabilities__cell { + display: table-cell; + } + @media (max-width: $break-xl) { + .source__cell { + display: none; + width: 0; + } + } + @media (max-width: $break-lg) { + .Vulnerabilities__cell { + display: none; + width: 0; + } + } + @media (max-width: $break-sm) { + .status__cell { + display: none; + width: 0; + } + } + } + } + } + } + } } From f0e390358595283874e8452738a93c7890294564 Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Tue, 14 May 2024 17:03:44 -0300 Subject: [PATCH 41/56] add boilerplate to host installation result outputs (#18988) Specified by Marko, this should be returned by the API: image # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Added/updated tests - [x] Manual QA for all new/changed functionality --- ee/server/service/software_installers.go | 1 + server/datastore/mysql/software_installers.go | 4 +- server/fleet/software_installer.go | 56 +++++++++++++ server/fleet/software_test.go | 81 +++++++++++++++++++ server/service/integration_enterprise_test.go | 36 ++++++--- 5 files changed, 165 insertions(+), 13 deletions(-) diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index e642b414c2..4fe8cafcf9 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -281,6 +281,7 @@ func (svc *Service) GetSoftwareInstallResults(ctx context.Context, resultUUID st return nil, err } + res.EnhanceOutputDetails() return res, nil } diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 40777f2354..427e251645 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -328,7 +328,9 @@ SELECT COALESCE(%s, '') AS status, si.filename AS software_package, h.team_id AS host_team_id, - hsi.user_id AS user_id + hsi.user_id AS user_id, + hsi.post_install_script_exit_code, + hsi.install_script_exit_code FROM host_software_installs hsi JOIN hosts h ON h.id = hsi.host_id diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index 52f083ce0e..1e8f47693b 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -168,6 +168,62 @@ type HostSoftwareInstallerResult struct { HostTeamID *uint `json:"-" db:"host_team_id"` // UserID is the user ID that requested the software installation on that host. UserID *uint `json:"-" db:"user_id"` + // InstallScriptExitCode is used internally to determine the output displayed to the user. + InstallScriptExitCode *int `json:"-" db:"install_script_exit_code"` + // PostInstallScriptExitCode is used internally to determine the output displayed to the user. + PostInstallScriptExitCode *int `json:"-" db:"post_install_script_exit_code"` +} + +const ( + SoftwareInstallerQueryFailCopy = "Query didn't return result or failed\nInstall stopped" + SoftwareInstallerQuerySuccessCopy = "Query returned result\nProceeding to install..." + SoftwareInstallerScriptsDisabledCopy = "Installing software...\nError: Scripts are disabled for this host. To run scripts, deploy the fleetd agent with --scripts-enabled." + SoftwareInstallerInstallFailCopy = "Installing software...\nFailed\n%s" + SoftwareInstallerInstallSuccessCopy = "Installing software...\nSuccess\n%s" + SoftwareInstallerPostInstallSuccessCopy = "Running script...\nExit code: 0 (Success)\n%s" + // TODO(roberto): this is not true, how do we know that the rollback script was successful? + SoftwareInstallerPostInstallFailCopy = `Running script... +Exit code: %d (Failed) +%s +Rolling back software install... +Rolled back successfully +` +) + +// EnhanceOutputDetails is used to add extra boilerplate/information to the +// output fields so they're easier to consume by users. +func (h *HostSoftwareInstallerResult) EnhanceOutputDetails() { + if h.Status == SoftwareInstallerPending { + return + } + + if h.PreInstallQueryOutput == "" { + h.PreInstallQueryOutput = SoftwareInstallerQueryFailCopy + return + } + h.PreInstallQueryOutput = SoftwareInstallerQuerySuccessCopy + + if h.InstallScriptExitCode == nil { + return + } + if *h.InstallScriptExitCode == -2 { + h.Output = SoftwareInstallerScriptsDisabledCopy + return + } else if *h.InstallScriptExitCode != 0 { + h.Output = fmt.Sprintf(SoftwareInstallerInstallFailCopy, h.Output) + return + } + h.Output = fmt.Sprintf(SoftwareInstallerInstallSuccessCopy, h.Output) + + if h.PostInstallScriptExitCode == nil { + return + } + if *h.PostInstallScriptExitCode != 0 { + h.PostInstallScriptOutput = fmt.Sprintf(SoftwareInstallerPostInstallFailCopy, *h.PostInstallScriptExitCode, h.PostInstallScriptOutput) + return + } + + h.PostInstallScriptOutput = fmt.Sprintf(SoftwareInstallerPostInstallSuccessCopy, h.PostInstallScriptOutput) } type HostSoftwareInstallerResultAuthz struct { diff --git a/server/fleet/software_test.go b/server/fleet/software_test.go index b0bfbb6962..c00708c0f7 100644 --- a/server/fleet/software_test.go +++ b/server/fleet/software_test.go @@ -3,6 +3,7 @@ package fleet import ( "testing" + "github.com/fleetdm/fleet/v4/server/ptr" "github.com/stretchr/testify/require" ) @@ -75,3 +76,83 @@ func TestParseSoftwareLastOpenedAtRowValue(t *testing.T) { require.NoError(t, err) require.NotZero(t, lastOpenedAt) } + +func TestEnhanceOutputDetails(t *testing.T) { + tests := []struct { + name string + hsr HostSoftwareInstallerResult + expectedPreInstallQueryOutput string + expectedOutput string + expectedPostInstallScriptOutput string + }{ + { + name: "pending status", + hsr: HostSoftwareInstallerResult{ + Status: SoftwareInstallerPending, + }, + expectedPreInstallQueryOutput: "", + expectedOutput: "", + expectedPostInstallScriptOutput: "", + }, + { + name: "non-pending status with empty PreInstallQueryOutput and successful install", + hsr: HostSoftwareInstallerResult{ + Status: SoftwareInstallerInstalled, + InstallScriptExitCode: ptr.Int(0), + PreInstallQueryOutput: "1", + }, + expectedPreInstallQueryOutput: "Query returned result\nProceeding to install...", + expectedOutput: "Installing software...\nSuccess\n", + expectedPostInstallScriptOutput: "", + }, + { + name: "non-pending status with empty PreInstallQueryOutput and failed install", + hsr: HostSoftwareInstallerResult{ + Status: SoftwareInstallerFailed, + InstallScriptExitCode: ptr.Int(1), + PreInstallQueryOutput: "1", + }, + expectedPreInstallQueryOutput: "Query returned result\nProceeding to install...", + expectedOutput: "Installing software...\nFailed\n", + expectedPostInstallScriptOutput: "", + }, + { + name: "non-pending status with non-empty PreInstallQueryOutput and disabled scripts", + hsr: HostSoftwareInstallerResult{ + Status: SoftwareInstallerFailed, + InstallScriptExitCode: ptr.Int(-2), + PreInstallQueryOutput: "1", + }, + expectedPreInstallQueryOutput: "Query returned result\nProceeding to install...", + expectedOutput: "Installing software...\nError: Scripts are disabled for this host. To run scripts, deploy the fleetd agent with --scripts-enabled.", + expectedPostInstallScriptOutput: "", + }, + { + name: "non-pending status with non-empty PreInstallQueryOutput and failed post install script", + hsr: HostSoftwareInstallerResult{ + Status: SoftwareInstallerInstalled, + InstallScriptExitCode: ptr.Int(0), + PostInstallScriptExitCode: ptr.Int(1), + PreInstallQueryOutput: "1", + PostInstallScriptOutput: "output!", + }, + expectedPreInstallQueryOutput: "Query returned result\nProceeding to install...", + expectedOutput: "Installing software...\nSuccess\n", + expectedPostInstallScriptOutput: `Running script... +Exit code: 1 (Failed) +output! +Rolling back software install... +Rolled back successfully +`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.hsr.EnhanceOutputDetails() + require.Equal(t, tt.expectedPreInstallQueryOutput, tt.hsr.PreInstallQueryOutput) + require.Equal(t, tt.expectedOutput, tt.hsr.Output) + require.Equal(t, tt.expectedPostInstallScriptOutput, tt.hsr.PostInstallScriptOutput) + }) + } +} diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index b92d7bfd4b..a431e2be76 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -9470,9 +9470,12 @@ func (s *integrationEnterpriseTestSuite) TestHostSoftwareInstallResult() { } type result struct { - HostID uint - InstallUUID string - Status fleet.SoftwareInstallerStatus + HostID uint + InstallUUID string + Status fleet.SoftwareInstallerStatus + Output string + PostInstallScriptOutput string + PreInstallQueryOutput string } checkResults := func(want result) { var resp getSoftwareInstallResultsResponse @@ -9481,6 +9484,9 @@ func (s *integrationEnterpriseTestSuite) TestHostSoftwareInstallResult() { assert.Equal(t, want.HostID, resp.Results.HostID) assert.Equal(t, want.InstallUUID, resp.Results.InstallUUID) assert.Equal(t, want.Status, resp.Results.Status) + assert.Equal(t, want.PreInstallQueryOutput, resp.Results.PreInstallQueryOutput) + assert.Equal(t, want.Output, resp.Results.Output) + assert.Equal(t, want.PostInstallScriptOutput, resp.Results.PostInstallScriptOutput) } s.Do("POST", "/api/fleet/orbit/software_install/result", @@ -9493,9 +9499,11 @@ func (s *integrationEnterpriseTestSuite) TestHostSoftwareInstallResult() { }`, *host.OrbitNodeKey, installUUIDs[0])), http.StatusNoContent) checkResults(result{ - HostID: host.ID, - InstallUUID: installUUIDs[0], - Status: fleet.SoftwareInstallerFailed, + HostID: host.ID, + InstallUUID: installUUIDs[0], + Status: fleet.SoftwareInstallerFailed, + PreInstallQueryOutput: fleet.SoftwareInstallerQuerySuccessCopy, + Output: fmt.Sprintf(fleet.SoftwareInstallerInstallFailCopy, "failed"), }) wantAct := fleet.ActivityTypeInstalledSoftware{ HostID: host.ID, @@ -9514,9 +9522,10 @@ func (s *integrationEnterpriseTestSuite) TestHostSoftwareInstallResult() { }`, *host.OrbitNodeKey, installUUIDs[1])), http.StatusNoContent) checkResults(result{ - HostID: host.ID, - InstallUUID: installUUIDs[1], - Status: fleet.SoftwareInstallerFailed, + HostID: host.ID, + InstallUUID: installUUIDs[1], + Status: fleet.SoftwareInstallerFailed, + PreInstallQueryOutput: fleet.SoftwareInstallerQueryFailCopy, }) wantAct.InstallUUID = installUUIDs[1] s.lastActivityOfTypeMatches(wantAct.ActivityName(), string(jsonMustMarshal(t, wantAct)), 0) @@ -9533,9 +9542,12 @@ func (s *integrationEnterpriseTestSuite) TestHostSoftwareInstallResult() { }`, *host.OrbitNodeKey, installUUIDs[2])), http.StatusNoContent) checkResults(result{ - HostID: host.ID, - InstallUUID: installUUIDs[2], - Status: fleet.SoftwareInstallerInstalled, + HostID: host.ID, + InstallUUID: installUUIDs[2], + Status: fleet.SoftwareInstallerInstalled, + PreInstallQueryOutput: fleet.SoftwareInstallerQuerySuccessCopy, + Output: fmt.Sprintf(fleet.SoftwareInstallerInstallSuccessCopy, "success"), + PostInstallScriptOutput: fmt.Sprintf(fleet.SoftwareInstallerPostInstallSuccessCopy, "ok"), }) wantAct.InstallUUID = installUUIDs[2] wantAct.Status = string(fleet.SoftwareInstallerInstalled) From 71c0026168d106dd1d4410bf5ec07c29ca196336 Mon Sep 17 00:00:00 2001 From: Dante Catalfamo <43040593+dantecatalfamo@users.noreply.github.com> Date: Tue, 14 May 2024 16:25:35 -0400 Subject: [PATCH 42/56] Orbit software installer flow (#18797) # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [x] Added/updated tests - [x] Manual QA for all new/changed functionality - For Orbit and Fleet Desktop changes: - [x] Manual QA must be performed in the three main OSs, macOS, Windows and Linux. - [x] Auto-update manual QA, from released version of component to new version (see [tools/tuf/test](../tools/tuf/test/README.md)). --------- Co-authored-by: Roberto Dip --- ee/server/service/software_installers.go | 6 + orbit/changes/18797-install-software | 1 + orbit/cmd/orbit/orbit.go | 4 + orbit/pkg/installer/installer.go | 266 +++++++++++ orbit/pkg/installer/installer_test.go | 423 ++++++++++++++++++ orbit/pkg/scripts/exec_nonwindows.go | 8 +- orbit/pkg/scripts/exec_nonwindows_test.go | 2 +- orbit/pkg/scripts/exec_windows.go | 3 +- orbit/pkg/scripts/scripts.go | 6 +- orbit/pkg/scripts/scripts_test.go | 2 +- pkg/file/file.go | 7 + pkg/file/file_test.go | 42 ++ pkg/file/xar.go | 6 +- server/datastore/mysql/software_installers.go | 6 +- server/fleet/orbit.go | 3 + server/service/base_client.go | 74 ++- server/service/orbit.go | 43 +- server/service/orbit_client.go | 48 +- server/service/orbit_test.go | 14 +- server/service/software_installers.go | 10 +- 20 files changed, 922 insertions(+), 52 deletions(-) create mode 100644 orbit/changes/18797-install-software create mode 100644 orbit/pkg/installer/installer.go create mode 100644 orbit/pkg/installer/installer_test.go diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index 4fe8cafcf9..a84617f10a 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -32,6 +32,12 @@ func (svc *Service) UploadSoftwareInstaller(ctx context.Context, payload *fleet. return fleet.ErrNoContext } + // make sure all scripts use unix-style newlines to prevent errors when + // running them, browsers use windows-style newlines, which breaks the + // shebang when the file is directly executed. + payload.InstallScript = file.Dos2UnixNewlines(payload.InstallScript) + payload.PostInstallScript = file.Dos2UnixNewlines(payload.PostInstallScript) + if _, err := svc.addMetadataToSoftwarePayload(ctx, payload); err != nil { return ctxerr.Wrap(ctx, err, "adding metadata to payload") } diff --git a/orbit/changes/18797-install-software b/orbit/changes/18797-install-software new file mode 100644 index 0000000000..a372c8a173 --- /dev/null +++ b/orbit/changes/18797-install-software @@ -0,0 +1 @@ +* Added ability to install software when requested by the Fleet server. Note that this is disabled unless the package was built with the `--enable-scripts` flag. diff --git a/orbit/cmd/orbit/orbit.go b/orbit/cmd/orbit/orbit.go index 787d296e7f..1413b6ae8f 100644 --- a/orbit/cmd/orbit/orbit.go +++ b/orbit/cmd/orbit/orbit.go @@ -23,6 +23,7 @@ import ( "github.com/fleetdm/fleet/v4/orbit/pkg/constant" "github.com/fleetdm/fleet/v4/orbit/pkg/execuser" "github.com/fleetdm/fleet/v4/orbit/pkg/insecure" + "github.com/fleetdm/fleet/v4/orbit/pkg/installer" "github.com/fleetdm/fleet/v4/orbit/pkg/keystore" "github.com/fleetdm/fleet/v4/orbit/pkg/logging" "github.com/fleetdm/fleet/v4/orbit/pkg/osquery" @@ -1148,6 +1149,9 @@ func main() { } } + softwareRunner := installer.NewRunner(orbitClient, r.ExtensionSocketPath(), scriptsEnabledFn) + orbitClient.RegisterConfigReceiver(softwareRunner) + // Install a signal handler ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/orbit/pkg/installer/installer.go b/orbit/pkg/installer/installer.go new file mode 100644 index 0000000000..46cd53fb2e --- /dev/null +++ b/orbit/pkg/installer/installer.go @@ -0,0 +1,266 @@ +package installer + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/fleetdm/fleet/v4/orbit/pkg/constant" + "github.com/fleetdm/fleet/v4/orbit/pkg/scripts" + "github.com/fleetdm/fleet/v4/pkg/file" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/osquery/osquery-go" + osquery_gen "github.com/osquery/osquery-go/gen/osquery" + "github.com/rs/zerolog/log" +) + +type QueryResponse = osquery_gen.ExtensionResponse +type QueryResponseStatus = osquery_gen.ExtensionStatus + +// Client defines the methods required for the API requests to the server. The +// fleet.OrbitClient type satisfies this interface. +type Client interface { + GetInstallerDetails(installID string) (*fleet.SoftwareInstallDetails, error) + DownloadSoftwareInstaller(installerID uint, downloadDir string) (string, error) + SaveInstallerResult(payload *fleet.HostSoftwareInstallResultPayload) error +} + +type QueryClient interface { + Query(context.Context, string) (*QueryResponse, error) +} + +type Runner struct { + OsqueryClient QueryClient + OrbitClient Client + + // osquerySocketPath is used to establish the osquery connection + // if it's ever lost or disconnected + osquerySocketPath string + + // tempDirFn is the function to call to get the temporary directory to use, + // inside of which the script-specific subdirectories will be created. If nil, + // the user's temp dir will be used (can be set to t.TempDir in tests). + tempDirFn func(dir, pattern string) (string, error) + + // execCmdFn can be set for tests to mock actual execution of the script. If + // nil, execCmd will be used, which has a different implementation on Windows + // and non-Windows platforms. + execCmdFn func(ctx context.Context, scriptPath string, env []string) ([]byte, int, error) + + // can be set for tests to replace os.RemoveAll, which is called to remove + // the script's temporary directory after execution. + removeAllFn func(string) error + + connectOsquery func(*Runner) error + + scriptsEnabled func() bool + + osqueryConnectionMutex sync.Mutex +} + +func NewRunner(client Client, socketPath string, scriptsEnabled func() bool) *Runner { + r := &Runner{ + OrbitClient: client, + osquerySocketPath: socketPath, + scriptsEnabled: scriptsEnabled, + } + + return r +} + +func (r *Runner) Run(config *fleet.OrbitConfig) error { + connectOsqueryFn := r.connectOsquery + if connectOsqueryFn == nil { + connectOsqueryFn = connectOsquery + } + + if err := connectOsqueryFn(r); err != nil { + return fmt.Errorf("software installer runner connecting to osquery: %w", err) + } + return r.run(context.Background(), config) +} + +func connectOsquery(r *Runner) error { + r.osqueryConnectionMutex.Lock() + defer r.osqueryConnectionMutex.Unlock() + + if r.OsqueryClient == nil { + osqueryClient, err := osquery.NewClient(r.osquerySocketPath, 10*time.Second) + if err != nil { + log.Err(err).Msg("establishing osquery connection for software install runner") + return err + } + + r.OsqueryClient = osqueryClient.Client + } + + return nil +} + +func (r *Runner) run(ctx context.Context, config *fleet.OrbitConfig) error { + var errs []error + for _, installerID := range config.Notifications.PendingSoftwareInstallerIDs { + if ctx.Err() != nil { + errs = append(errs, ctx.Err()) + break + } + payload, err := r.installSoftware(ctx, installerID) + if err != nil { + errs = append(errs, err) + if payload == nil { + continue + } + } + if err := r.OrbitClient.SaveInstallerResult(payload); err != nil { + errs = append(errs, fmt.Errorf("saving software install results: %w", err)) + } + } + if len(errs) != 0 { + return errors.Join(errs...) + } + + return nil +} + +func (r *Runner) preConditionCheck(ctx context.Context, query string) (bool, string, error) { + res, err := r.OsqueryClient.Query(ctx, query) + if err != nil { + return false, "", fmt.Errorf("precondition check: %w", err) + } + + if res.Status == nil { + return false, "", errors.New("no query status") + } + + if res.Status.Code != 0 { + // TODO(roberto): we can't return the error as the + // result because the back-end considers any non-empty + // string as a success. + // return false, fmt.Sprintf("osqueryd returned error (%d): %s", res.Status.Code, res.Status.Message), fmt.Errorf("non-zero query status: %d \"%s\"", res.Status.Code, res.Status.Message) + return false, "", fmt.Errorf("non-zero query status: %d \"%s\"", res.Status.Code, res.Status.Message) + } + + if len(res.Response) == 0 { + return false, "", nil + } + + response, err := json.Marshal(res.Response) + if err != nil { + return false, "", fmt.Errorf("marshalling query response: %w", err) + } + + return true, string(response), nil +} + +func (r *Runner) installSoftware(ctx context.Context, installID string) (*fleet.HostSoftwareInstallResultPayload, error) { + installer, err := r.OrbitClient.GetInstallerDetails(installID) + if err != nil { + return nil, fmt.Errorf("fetching software installer details: %w", err) + } + + payload := &fleet.HostSoftwareInstallResultPayload{} + payload.InstallUUID = installID + + if installer.PreInstallCondition != "" { + shouldInstall, output, err := r.preConditionCheck(ctx, installer.PreInstallCondition) + payload.PreInstallConditionOutput = &output + if err != nil { + return payload, err + } + + if !shouldInstall { + return payload, nil + } + } + + if !r.scriptsEnabled() { + // fleetctl knows that -2 means script was disabled on host + payload.InstallScriptExitCode = ptr.Int(-2) + payload.InstallScriptOutput = ptr.String("Scripts are disabled") + return payload, nil + } + + tmpDirFn := r.tempDirFn + if tmpDirFn == nil { + tmpDirFn = os.MkdirTemp + } + tmpDir, err := tmpDirFn("", "") + if err != nil { + return payload, fmt.Errorf("creating temporary directory: %w", err) + } + + installerPath, err := r.OrbitClient.DownloadSoftwareInstaller(installer.InstallerID, tmpDir) + if err != nil { + return payload, err + } + + // remove tmp directory and installer + defer func() { + removeAllFn := r.removeAllFn + if removeAllFn == nil { + removeAllFn = os.RemoveAll + } + err := removeAllFn(tmpDir) + if err != nil { + log.Err(err) + } + }() + + installOutput, installExitCode, err := r.runInstallerScript(ctx, installer.InstallScript, installerPath, "install-script") + payload.InstallScriptOutput = &installOutput + payload.InstallScriptExitCode = &installExitCode + if err != nil { + return payload, err + } + + if installer.PostInstallScript != "" { + postOutput, postExitCode, postErr := r.runInstallerScript(ctx, installer.PostInstallScript, installerPath, "post-install-script") + payload.PostInstallScriptOutput = &postOutput + payload.PostInstallScriptExitCode = &postExitCode + + if postErr != nil || postExitCode != 0 { + log.Info().Msgf("installation of %s failed, attempting rollback. Exit code: %d, error: %s", installerPath, postExitCode, postErr) + uninstallScript := file.GetRemoveScript(installerPath) + uninstallOutput, uninstallExitCode, uninstallErr := r.runInstallerScript(ctx, uninstallScript, installerPath, "rollback-script") + log.Info().Msgf( + "rollback staus: exit code: %d, error: %s, output: %s", + uninstallExitCode, uninstallErr, uninstallOutput, + ) + + return payload, postErr + } + } + + return payload, nil +} + +func (r *Runner) runInstallerScript(ctx context.Context, scriptContents string, installerPath string, fileName string) (string, int, error) { + // run script in installer directory + installerDir := filepath.Dir(installerPath) + scriptPath := filepath.Join(installerDir, fileName) + if err := os.WriteFile(scriptPath, []byte(scriptContents), constant.DefaultFileMode); err != nil { + return "", -1, fmt.Errorf("writing script: %w", err) + } + + execFn := r.execCmdFn + if execFn == nil { + execFn = scripts.ExecCmd + } + + env := os.Environ() + installerPathEnv := fmt.Sprintf("INSTALLER_PATH=%s", installerPath) + env = append(env, installerPathEnv) + + output, exitCode, err := execFn(ctx, scriptPath, env) + if err != nil { + return string(output), exitCode, err + } + + return string(output), exitCode, nil +} diff --git a/orbit/pkg/installer/installer_test.go b/orbit/pkg/installer/installer_test.go new file mode 100644 index 0000000000..7b61290269 --- /dev/null +++ b/orbit/pkg/installer/installer_test.go @@ -0,0 +1,423 @@ +package installer + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strconv" + "testing" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" + osquery_gen "github.com/osquery/osquery-go/gen/osquery" + "github.com/stretchr/testify/require" +) + +type TestOrbitClient struct { + downloadInstallerFn func(uint, string) (string, error) + getInstallerDetailsFn func(string) (*fleet.SoftwareInstallDetails, error) + saveInstallerResultFn func(*fleet.HostSoftwareInstallResultPayload) error +} + +func (oc *TestOrbitClient) DownloadSoftwareInstaller(installerID uint, downloadDir string) (string, error) { + return oc.downloadInstallerFn(installerID, downloadDir) +} + +func (oc *TestOrbitClient) GetInstallerDetails(installId string) (*fleet.SoftwareInstallDetails, error) { + return oc.getInstallerDetailsFn(installId) +} + +func (oc *TestOrbitClient) SaveInstallerResult(payload *fleet.HostSoftwareInstallResultPayload) error { + return oc.saveInstallerResultFn(payload) +} + +type TestQueryClient struct { + queryFn func(context.Context, string) (*QueryResponse, error) +} + +func (qc *TestQueryClient) Query(ctx context.Context, query string) (*QueryResponse, error) { + return qc.queryFn(ctx, query) +} + +func TestRunInstallScript(t *testing.T) { + oc := &TestOrbitClient{} + r := Runner{OrbitClient: oc, scriptsEnabled: func() bool { return true }} + + var executedScriptPath string + var executed bool + var executedEnv []string + execCmd := func(ctx context.Context, spath string, env []string) ([]byte, int, error) { + executed = true + executedScriptPath = spath + executedEnv = env + return []byte("bye"), 2, nil + } + r.execCmdFn = execCmd + + installerDir := t.TempDir() + installerPath := filepath.Join(installerDir, "installer.pkg") + + output, exitCode, err := r.runInstallerScript(context.Background(), "hello", installerPath, "foo") + + require.Equal(t, executedScriptPath, filepath.Join(installerDir, "foo")) + require.Contains(t, executedScriptPath, installerDir) + require.True(t, executed) + + require.Nil(t, err) + require.Equal(t, "bye", output) + require.Equal(t, 2, exitCode) + require.Contains(t, executedEnv, "INSTALLER_PATH="+installerPath) +} + +func TestPreconditionCheck(t *testing.T) { + qc := &TestQueryClient{} + r := &Runner{OsqueryClient: qc, scriptsEnabled: func() bool { return true }} + + qc.queryFn = func(ctx context.Context, s string) (*QueryResponse, error) { + qr := &QueryResponse{ + Status: &osquery_gen.ExtensionStatus{}, + } + + switch s { + case "empty": + case "error": + return nil, errors.New("something bad") + case "badstatus": + qr.Status.Code = 1 + qr.Status.Message = "something bad" + case "nostatus": + qr.Status = nil + case "response": + row := make(map[string]string) + row["key"] = "value" + qr.Response = append(qr.Response, row) + default: + t.Error("invalid query test case") + } + + return qr, nil + } + + ctx := context.Background() + + // empty query response + success, output, err := r.preConditionCheck(ctx, "empty") + require.False(t, success) + require.Nil(t, err) + require.Equal(t, "", output) + + success, output, err = r.preConditionCheck(ctx, "response") + require.True(t, success) + require.Nil(t, err) + require.Equal(t, "[{\"key\":\"value\"}]", output) + + success, output, err = r.preConditionCheck(ctx, "error") + require.False(t, success) + require.Error(t, err) + require.Equal(t, "", output) + + success, output, err = r.preConditionCheck(ctx, "badstatus") + require.False(t, success) + require.Error(t, err) + require.Equal(t, "", output) + + success, output, err = r.preConditionCheck(ctx, "nostatus") + require.False(t, success) + require.Error(t, err) + require.Equal(t, "", output) +} + +func TestInstallerRun(t *testing.T) { + oc := &TestOrbitClient{} + + var getInstallerDetailsFnCalled bool + var installIdRequested string + installDetails := &fleet.SoftwareInstallDetails{ + ExecutionID: "exec1", + InstallerID: 1337, + PreInstallCondition: "SELECT 1", + InstallScript: "script1", + PostInstallScript: "script2", + } + getInstallerDetailsDefaultFn := func(installID string) (*fleet.SoftwareInstallDetails, error) { + getInstallerDetailsFnCalled = true + installIdRequested = installID + return installDetails, nil + } + oc.getInstallerDetailsFn = getInstallerDetailsDefaultFn + + var downloadInstallerFnCalled bool + downloadInstallerDefaultFn := func(installerID uint, downloadDir string) (string, error) { + downloadInstallerFnCalled = true + return filepath.Join(downloadDir, strconv.Itoa(int(installerID))+".pkg"), nil + } + oc.downloadInstallerFn = downloadInstallerDefaultFn + + var savedInstallerResult *fleet.HostSoftwareInstallResultPayload + oc.saveInstallerResultFn = func(hsirp *fleet.HostSoftwareInstallResultPayload) error { + savedInstallerResult = hsirp + return nil + } + + resetTestOrbitClient := func() { + getInstallerDetailsFnCalled = false + installIdRequested = "" + oc.getInstallerDetailsFn = getInstallerDetailsDefaultFn + installDetails = &fleet.SoftwareInstallDetails{ + ExecutionID: "exec1", + InstallerID: 1337, + PreInstallCondition: "SELECT 1", + InstallScript: "script1", + PostInstallScript: "script2", + } + downloadInstallerFnCalled = false + oc.downloadInstallerFn = downloadInstallerDefaultFn + savedInstallerResult = nil + } + + q := &TestQueryClient{} + + var queryFnCalled bool + var queryFnQuery string + queryFnResMap := make(map[string]string, 0) + queryFnResMap["col"] = "true" + queryFnResArr := []map[string]string{queryFnResMap} + queryFnResStatus := &QueryResponseStatus{} + queryFnResponse := &QueryResponse{ + Response: queryFnResArr, + Status: queryFnResStatus, + } + queryDefaultFn := func(ctx context.Context, query string) (*QueryResponse, error) { + queryFnQuery = query + queryFnCalled = true + return queryFnResponse, nil + } + q.queryFn = queryDefaultFn + + resetTestQueryClient := func() { + queryFnCalled = false + queryFnQuery = "" + queryFnResMap = make(map[string]string, 0) + queryFnResMap["col"] = "true" + queryFnResArr = []map[string]string{queryFnResMap} + queryFnResStatus = &QueryResponseStatus{} + queryFnResponse = &QueryResponse{ + Response: queryFnResArr, + Status: queryFnResStatus, + } + q.queryFn = queryDefaultFn + } + + r := &Runner{ + OrbitClient: oc, + OsqueryClient: q, + scriptsEnabled: func() bool { return true }, + } + + var execCalled bool + var executedScripts []string + var execEnv []string + var execErr error + execOutput := []byte("execOutput") + execExitCode := 0 + execCmdDefaultFn := func(ctx context.Context, scriptPath string, env []string) ([]byte, int, error) { + execCalled = true + execEnv = env + executedScripts = append(executedScripts, scriptPath) + return execOutput, execExitCode, execErr + } + r.execCmdFn = execCmdDefaultFn + + var tmpDirFnCalled bool + var tmpDir string + r.tempDirFn = func(dir, pattern string) (string, error) { + tmpDirFnCalled = true + tmpDir = os.TempDir() + return tmpDir, nil + } + + var removeAllFnCalled bool + var removedDir string + r.removeAllFn = func(s string) error { + removedDir = s + removeAllFnCalled = true + return nil + } + + resetRunner := func() { + execCalled = false + executedScripts = nil + execEnv = nil + execOutput = []byte("execOutput") + execExitCode = 0 + execErr = nil + r.execCmdFn = execCmdDefaultFn + + tmpDirFnCalled = false + tmpDir = "" + } + + var config fleet.OrbitConfig + config.Notifications.PendingSoftwareInstallerIDs = []string{installDetails.ExecutionID} + + resetConfig := func() { + config.Notifications.PendingSoftwareInstallerIDs = []string{installDetails.ExecutionID} + } + + resetAll := func() { + resetTestOrbitClient() + resetTestQueryClient() + resetRunner() + resetConfig() + } + + t.Run("everything good", func(t *testing.T) { + resetAll() + + err := r.run(context.Background(), &config) + require.NoError(t, err) + + require.True(t, removeAllFnCalled) + require.Equal(t, tmpDir, removedDir) + + require.True(t, tmpDirFnCalled) + + require.True(t, execCalled) + require.Contains(t, executedScripts, filepath.Join(tmpDir, "install-script")) + require.Contains(t, executedScripts, filepath.Join(tmpDir, "post-install-script")) + require.Contains(t, execEnv, "INSTALLER_PATH="+filepath.Join(tmpDir, strconv.Itoa(int(installDetails.InstallerID))+".pkg")) + + require.True(t, queryFnCalled) + require.Equal(t, installDetails.PreInstallCondition, queryFnQuery) + + require.NotNil(t, savedInstallerResult) + require.Equal(t, execExitCode, *savedInstallerResult.InstallScriptExitCode) + require.Equal(t, string(execOutput), *savedInstallerResult.InstallScriptOutput) + require.Equal(t, execExitCode, *savedInstallerResult.PostInstallScriptExitCode) + require.Equal(t, string(execOutput), *savedInstallerResult.PostInstallScriptOutput) + require.Equal(t, installDetails.ExecutionID, savedInstallerResult.InstallUUID) + + require.True(t, downloadInstallerFnCalled) + + require.True(t, getInstallerDetailsFnCalled) + require.Equal(t, installDetails.ExecutionID, installIdRequested) + }) + + t.Run("precondition negative", func(t *testing.T) { + resetAll() + + queryFnResponse.Response = []map[string]string{} + + err := r.run(context.Background(), &config) + require.NoError(t, err) + + require.False(t, downloadInstallerFnCalled) + require.False(t, execCalled) + require.True(t, removeAllFnCalled) + require.NotNil(t, savedInstallerResult) + require.Equal(t, installDetails.ExecutionID, savedInstallerResult.InstallUUID) + require.Nil(t, savedInstallerResult.InstallScriptExitCode) + require.Nil(t, savedInstallerResult.InstallScriptOutput) + require.Nil(t, savedInstallerResult.PostInstallScriptExitCode) + require.Nil(t, savedInstallerResult.PostInstallScriptOutput) + }) + + t.Run("failed install script", func(t *testing.T) { + resetAll() + + execErr = &exec.ExitError{} + execExitCode = 2 + + err := r.run(context.Background(), &config) + require.Error(t, err) + + require.True(t, downloadInstallerFnCalled) + require.True(t, execCalled) + require.True(t, removeAllFnCalled) + require.NotNil(t, savedInstallerResult) + require.Equal(t, installDetails.ExecutionID, savedInstallerResult.InstallUUID) + require.Equal(t, 2, *savedInstallerResult.InstallScriptExitCode) + require.Equal(t, string(execOutput), *savedInstallerResult.InstallScriptOutput) + require.Nil(t, savedInstallerResult.PostInstallScriptExitCode) + require.Nil(t, savedInstallerResult.PostInstallScriptOutput) + }) + + t.Run("failed post install script", func(t *testing.T) { + resetAll() + + r.execCmdFn = func(ctx context.Context, scriptPath string, env []string) ([]byte, int, error) { + execCalled = true + execEnv = env + executedScripts = append(executedScripts, scriptPath) + // bad exit on the post-install script + if len(executedScripts) == 2 { + return execOutput, 1, &exec.ExitError{} + } + return execOutput, execExitCode, execErr + } + + err := r.run(context.Background(), &config) + require.Error(t, err) + + require.True(t, downloadInstallerFnCalled) + require.True(t, execCalled) + require.True(t, removeAllFnCalled) + require.NotNil(t, savedInstallerResult) + require.Equal(t, installDetails.ExecutionID, savedInstallerResult.InstallUUID) + require.Equal(t, 0, *savedInstallerResult.InstallScriptExitCode) + require.Equal(t, string(execOutput), *savedInstallerResult.InstallScriptOutput) + require.Equal(t, 1, *savedInstallerResult.PostInstallScriptExitCode) + require.Equal(t, string(execOutput), *savedInstallerResult.PostInstallScriptOutput) + }) +} + +func TestScriptsDisabled(t *testing.T) { + oc := &TestOrbitClient{} + qc := &TestQueryClient{} + r := &Runner{ + OrbitClient: oc, + OsqueryClient: qc, + scriptsEnabled: func() bool { return false }, + } + + qc.queryFn = func(ctx context.Context, s string) (*QueryResponse, error) { + + queryFnResMap := make(map[string]string, 0) + queryFnResMap["col"] = "true" + queryFnResArr := []map[string]string{queryFnResMap} + queryFnResStatus := &QueryResponseStatus{} + return &QueryResponse{ + Response: queryFnResArr, + Status: queryFnResStatus, + }, nil + } + + var getInstallerDetailsFnCalled bool + var installIdRequested string + installDetails := &fleet.SoftwareInstallDetails{ + ExecutionID: "exec1", + InstallerID: 1337, + PreInstallCondition: "SELECT 1", + InstallScript: "script1", + PostInstallScript: "script2", + } + getInstallerDetailsDefaultFn := func(installID string) (*fleet.SoftwareInstallDetails, error) { + getInstallerDetailsFnCalled = true + installIdRequested = installID + return installDetails, nil + } + oc.getInstallerDetailsFn = getInstallerDetailsDefaultFn + + out, err := r.installSoftware(context.Background(), "1") + require.NoError(t, err) + require.EqualValues(t, &fleet.HostSoftwareInstallResultPayload{ + InstallUUID: "1", + InstallScriptExitCode: ptr.Int(-2), + InstallScriptOutput: ptr.String("Scripts are disabled"), + PreInstallConditionOutput: ptr.String(`[{"col":"true"}]`), + }, out) + require.True(t, getInstallerDetailsFnCalled) + require.Equal(t, "1", installIdRequested) +} diff --git a/orbit/pkg/scripts/exec_nonwindows.go b/orbit/pkg/scripts/exec_nonwindows.go index 158333a17f..069e2d676b 100644 --- a/orbit/pkg/scripts/exec_nonwindows.go +++ b/orbit/pkg/scripts/exec_nonwindows.go @@ -12,7 +12,7 @@ import ( "github.com/fleetdm/fleet/v4/server/fleet" ) -func execCmd(ctx context.Context, scriptPath string) (output []byte, exitCode int, err error) { +func ExecCmd(ctx context.Context, scriptPath string, env []string) (output []byte, exitCode int, err error) { // initialize to -1 in case the process never starts exitCode = -1 @@ -28,13 +28,17 @@ func execCmd(ctx context.Context, scriptPath string) (output []byte, exitCode in cmd := exec.CommandContext(ctx, "/bin/sh", scriptPath) if directExecute { - err = os.Chmod(scriptPath, 0766) + err = os.Chmod(scriptPath, 0700) if err != nil { return nil, -1, ctxerr.Wrapf(ctx, err, "marking script as executable %s", scriptPath) } cmd = exec.CommandContext(ctx, scriptPath) } + if env != nil { + cmd.Env = env + } + cmd.Dir = filepath.Dir(scriptPath) output, err = cmd.CombinedOutput() if cmd.ProcessState != nil { diff --git a/orbit/pkg/scripts/exec_nonwindows_test.go b/orbit/pkg/scripts/exec_nonwindows_test.go index 4965b08838..891273c4e0 100644 --- a/orbit/pkg/scripts/exec_nonwindows_test.go +++ b/orbit/pkg/scripts/exec_nonwindows_test.go @@ -71,7 +71,7 @@ func TestExecCmdNonWindows(t *testing.T) { err := os.WriteFile(scriptPath, []byte(tc.contents), os.ModePerm) require.NoError(t, err) - output, exitCode, err := execCmd(context.Background(), scriptPath) + output, exitCode, err := ExecCmd(context.Background(), scriptPath, nil) require.Equal(t, tc.output, strings.TrimSpace(string(output))) require.Equal(t, tc.exitCode, exitCode) require.ErrorIs(t, err, tc.error) diff --git a/orbit/pkg/scripts/exec_windows.go b/orbit/pkg/scripts/exec_windows.go index a2619a7e6f..f0d58e17ef 100644 --- a/orbit/pkg/scripts/exec_windows.go +++ b/orbit/pkg/scripts/exec_windows.go @@ -8,12 +8,13 @@ import ( "path/filepath" ) -func execCmd(ctx context.Context, scriptPath string) (output []byte, exitCode int, err error) { +func ExecCmd(ctx context.Context, scriptPath string, env []string) (output []byte, exitCode int, err error) { // initialize to -1 in case the process never starts exitCode = -1 // for Windows, we execute the file with powershell. cmd := exec.CommandContext(ctx, "powershell", "-MTA", "-ExecutionPolicy", "Bypass", "-File", scriptPath) + cmd.Env = env cmd.Dir = filepath.Dir(scriptPath) output, err = cmd.CombinedOutput() if cmd.ProcessState != nil { diff --git a/orbit/pkg/scripts/scripts.go b/orbit/pkg/scripts/scripts.go index 547af0fd40..397e67a916 100644 --- a/orbit/pkg/scripts/scripts.go +++ b/orbit/pkg/scripts/scripts.go @@ -39,7 +39,7 @@ type Runner struct { // execCmdFn can be set for tests to mock actual execution of the script. If // nil, execCmd will be used, which has a different implementation on Windows // and non-Windows platforms. - execCmdFn func(ctx context.Context, scriptPath string) ([]byte, int, error) + execCmdFn func(ctx context.Context, scriptPath string, env []string) ([]byte, int, error) // can be set for tests to replace os.RemoveAll, which is called to remove // the script's temporary directory after execution. @@ -117,10 +117,10 @@ func (r *Runner) runOne(script *fleet.HostScriptResult) (finalErr error) { execCmdFn := r.execCmdFn if execCmdFn == nil { - execCmdFn = execCmd + execCmdFn = ExecCmd } start := time.Now() - output, exitCode, execErr := execCmdFn(ctx, scriptFile) + output, exitCode, execErr := execCmdFn(ctx, scriptFile, nil) duration := time.Since(start) // report the output or the error diff --git a/orbit/pkg/scripts/scripts_test.go b/orbit/pkg/scripts/scripts_test.go index e7eb22f4f8..fa3a5cea3d 100644 --- a/orbit/pkg/scripts/scripts_test.go +++ b/orbit/pkg/scripts/scripts_test.go @@ -349,7 +349,7 @@ type mockExecCmd struct { execFn func() ([]byte, int, error) } -func (m *mockExecCmd) run(ctx context.Context, scriptPath string) ([]byte, int, error) { +func (m *mockExecCmd) run(ctx context.Context, scriptPath string, env []string) ([]byte, int, error) { m.count++ if m.execFn != nil { return m.execFn() diff --git a/pkg/file/file.go b/pkg/file/file.go index c4fecc6fbd..80e620187b 100644 --- a/pkg/file/file.go +++ b/pkg/file/file.go @@ -12,6 +12,7 @@ import ( "os" "path" "path/filepath" + "strings" "github.com/fleetdm/fleet/v4/pkg/secure" ) @@ -130,6 +131,12 @@ func Exists(path string) (bool, error) { return info.Mode().IsRegular(), nil } +// Dos2UnixNewlines takes a string containing Windows-style newlines (\r\n) and +// converts them to Unix-style newlines (\n). It returns the converted string. +func Dos2UnixNewlines(s string) string { + return strings.ReplaceAll(s, "\r\n", "\n") +} + func ExtractFilenameFromURLPath(p string, defaultExtension string) string { u, err := url.Parse(p) if err != nil { diff --git a/pkg/file/file_test.go b/pkg/file/file_test.go index aa4fe2b6a7..69c92d1a8b 100644 --- a/pkg/file/file_test.go +++ b/pkg/file/file_test.go @@ -140,6 +140,48 @@ func TestExtractInstallerMetadata(t *testing.T) { } } +func TestDos2UnixNewlines(t *testing.T) { + testCases := []struct { + name string + input string + expected string + }{ + { + name: "No newlines", + input: "Hello World", + expected: "Hello World", + }, + { + name: "Single Windows newline", + input: "Hello\r\nWorld", + expected: "Hello\nWorld", + }, + { + name: "Multiple Windows newlines", + input: "Hello\r\nWorld\r\nTest", + expected: "Hello\nWorld\nTest", + }, + { + name: "Mixed newlines", + input: "Hello\r\nWorld\nTest", + expected: "Hello\nWorld\nTest", + }, + { + name: "All unix", + input: "Hello\nWorld\nTest", + expected: "Hello\nWorld\nTest", + }, + } + + // Execute each test case + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := file.Dos2UnixNewlines(tc.input) + require.Equal(t, tc.expected, result) + }) + } +} + func TestExtractFilenameFromURLPath(t *testing.T) { cases := []struct { in string diff --git a/pkg/file/xar.go b/pkg/file/xar.go index 3ec779345c..85eade2310 100644 --- a/pkg/file/xar.go +++ b/pkg/file/xar.go @@ -20,6 +20,7 @@ package file import ( "bytes" + "compress/bzip2" "compress/zlib" "crypto" "crypto/sha256" @@ -165,9 +166,10 @@ func ExtractXARMetadata(r io.Reader) (name, version string, shaSum []byte, err e } defer zr.Close() fileReader = zr - - // TODO(mna): obviously, we may need to support more decompression methods here... + } else if strings.Contains(f.Data.Encoding.Style, "x-bzip2") { + fileReader = bzip2.NewReader(fileReader) } + // TODO: what other compression methods are supported? contents, err := io.ReadAll(fileReader) if err != nil { diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 427e251645..9e6bb10398 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -41,9 +41,9 @@ func (ds *Datastore) GetSoftwareInstallDetails(ctx context.Context, executionId hsi.host_id AS host_id, hsi.execution_id AS execution_id, hsi.software_installer_id AS installer_id, - si.pre_install_query AS pre_install_condition, + COALESCE(si.pre_install_query, '') AS pre_install_condition, inst.contents AS install_script, - pisnt.contents AS post_install_script + COALESCE(pisnt.contents, '') AS post_install_script FROM host_software_installs hsi INNER JOIN @@ -63,7 +63,7 @@ func (ds *Datastore) GetSoftwareInstallDetails(ctx context.Context, executionId if err == sql.ErrNoRows { return nil, ctxerr.Wrap(ctx, notFound("SoftwareInstallerDetails").WithName(executionId), "get software installer details") } - return nil, ctxerr.Wrap(ctx, err, "list pending software installs") + return nil, ctxerr.Wrap(ctx, err, "get software install details") } return result, nil } diff --git a/server/fleet/orbit.go b/server/fleet/orbit.go index 077032d4b3..606cedf5f1 100644 --- a/server/fleet/orbit.go +++ b/server/fleet/orbit.go @@ -33,6 +33,9 @@ type OrbitConfigNotifications struct { // EnforceBitLockerEncryption is sent as true if Windows MDM is // enabled and the device should encrypt its disk volumes with BitLocker. EnforceBitLockerEncryption bool `json:"enforce_bitlocker_encryption,omitempty"` + + // PendingSoftwareInstallerIDs contains a list of software install_ids queued for installation + PendingSoftwareInstallerIDs []string `json:"pending_software_installer_ids,omitempty"` } type OrbitConfig struct { diff --git a/server/service/base_client.go b/server/service/base_client.go index dc771b74a0..194fa315ff 100644 --- a/server/service/base_client.go +++ b/server/service/base_client.go @@ -7,13 +7,16 @@ import ( "errors" "fmt" "io" + "mime" "net/http" "net/url" "os" + "path/filepath" "strings" "github.com/fleetdm/fleet/v4/pkg/fleethttp" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/google/uuid" ) var errInvalidScheme = errors.New("address must start with https:// for remote connections") @@ -63,17 +66,23 @@ func (bc *baseClient) parseResponse(verb, path string, response *http.Response, bc.setServerCapabilities(response) - if responseDest != nil && response.StatusCode != http.StatusNoContent { - b, err := io.ReadAll(response.Body) - if err != nil { - return fmt.Errorf("reading response body: %w", err) - } - if err := json.Unmarshal(b, &responseDest); err != nil { - return fmt.Errorf("decode %s %s response: %w, body: %s", verb, path, err, b) - } - if e, ok := responseDest.(errorer); ok { - if e.error() != nil { - return fmt.Errorf("%s %s error: %w", verb, path, e.error()) + if responseDest != nil { + if e, ok := responseDest.(bodyHandler); ok { + if err := e.Handle(response); err != nil { + return fmt.Errorf("%s %s error with custom body handler contents: %w", verb, path, err) + } + } else if response.StatusCode != http.StatusNoContent { + b, err := io.ReadAll(response.Body) + if err != nil { + return fmt.Errorf("reading response body: %w", err) + } + if err := json.Unmarshal(b, &responseDest); err != nil { + return fmt.Errorf("decode %s %s response: %w, body: %s", verb, path, err, b) + } + if e, ok := responseDest.(errorer); ok { + if e.error() != nil { + return fmt.Errorf("%s %s error: %w", verb, path, e.error()) + } } } } @@ -182,3 +191,46 @@ func newBaseClient( } return client, nil } + +type bodyHandler interface { + Handle(*http.Response) error +} + +type FileResponse struct { + DestPath string + destFilePath string +} + +func (f *FileResponse) Handle(resp *http.Response) error { + _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition")) + if err != nil { + return fmt.Errorf("parsing media type from response header: %w", err) + } + + filename := params["filename"] + if filename == "" { + filename = uuid.NewString() + } + + f.destFilePath = filepath.Join(f.DestPath, filename) + destFile, err := os.Create(f.destFilePath) + if err != nil { + return fmt.Errorf("creating file: %w", err) + } + defer destFile.Close() + + _, err = io.Copy(destFile, resp.Body) + if err != nil { + return fmt.Errorf("copying from http stream to file: %w", err) + } + + if err := destFile.Close(); err != nil { + return fmt.Errorf("closing file after copy: %w", err) + } + + return nil +} + +func (f *FileResponse) GetFilePath() string { + return f.destFilePath +} diff --git a/server/service/orbit.go b/server/service/orbit.go index eb6297a725..3581957078 100644 --- a/server/service/orbit.go +++ b/server/service/orbit.go @@ -241,6 +241,14 @@ func (svc *Service) GetOrbitConfig(ctx context.Context) (fleet.OrbitConfig, erro } } + pendingInstalls, err := svc.ds.ListPendingSoftwareInstalls(ctx, host.ID) + if err != nil { + return fleet.OrbitConfig{}, err + } + if len(pendingInstalls) > 0 { + notifs.PendingSoftwareInstallerIDs = pendingInstalls + } + // team ID is not nil, get team specific flags and options if host.TeamID != nil { teamAgentOptions, err := svc.ds.TeamAgentOptions(ctx, *host.TeamID) @@ -761,21 +769,7 @@ func (r *orbitGetSoftwareInstallRequest) setOrbitNodeKey(nodeKey string) { r.OrbitNodeKey = nodeKey } -// Download Orbit software installer request -///////////////////////////////////////////////////////////////////////////////// - -type orbitDownloadSoftwareInstallerRequest struct { - Alt string `query:"alt"` - OrbitNodeKey string `json:"orbit_node_key"` - InstallerID uint `json:"installer_id"` -} - // interface implementation required by the OrbitClient -func (r *orbitDownloadSoftwareInstallerRequest) setOrbitNodeKey(nodeKey string) { - r.OrbitNodeKey = nodeKey -} - -// interface implementation required by orbit authentication func (r *orbitGetSoftwareInstallRequest) orbitHostNodeKey() string { return r.OrbitNodeKey } @@ -818,6 +812,21 @@ func (svc *Service) GetSoftwareInstallDetails(ctx context.Context, installUUID s return details, nil } +// Download Orbit software installer request +///////////////////////////////////////////////////////////////////////////////// + +type orbitDownloadSoftwareInstallerRequest struct { + Alt string `query:"alt"` + OrbitNodeKey string `json:"orbit_node_key"` + InstallerID uint `json:"installer_id"` +} + +// interface implementation required by the OrbitClient +func (r *orbitDownloadSoftwareInstallerRequest) setOrbitNodeKey(nodeKey string) { + r.OrbitNodeKey = nodeKey +} + +// interface implementation required by orbit authentication func (r *orbitDownloadSoftwareInstallerRequest) orbitHostNodeKey() string { return r.OrbitNodeKey } @@ -828,14 +837,14 @@ func orbitDownloadSoftwareInstallerEndpoint(ctx context.Context, request interfa downloadRequested := req.Alt == "media" if !downloadRequested { // TODO: confirm error handling - return downloadSoftwareInstallerResponse{Err: &fleet.BadRequestError{Message: "only alt=media is supported"}}, nil + return orbitDownloadSoftwareInstallerResponse{Err: &fleet.BadRequestError{Message: "only alt=media is supported"}}, nil } p, err := svc.OrbitDownloadSoftwareInstaller(ctx, req.InstallerID) if err != nil { - return downloadSoftwareInstallerResponse{Err: err}, nil + return orbitDownloadSoftwareInstallerResponse{Err: err}, nil } - return downloadSoftwareInstallerResponse{payload: p}, nil + return orbitDownloadSoftwareInstallerResponse{payload: p}, nil } func (svc *Service) OrbitDownloadSoftwareInstaller(ctx context.Context, installerID uint) (*fleet.DownloadSoftwareInstallerPayload, error) { diff --git a/server/service/orbit_client.go b/server/service/orbit_client.go index 96323f3b99..619de10ecc 100644 --- a/server/service/orbit_client.go +++ b/server/service/orbit_client.go @@ -10,6 +10,7 @@ import ( "io/fs" "net" "net/http" + "net/url" "os" "path/filepath" "runtime" @@ -75,9 +76,14 @@ func (oc *OrbitClient) request(verb string, path string, params interface{}, res } } + parsedURL, err := url.Parse(path) + if err != nil { + return fmt.Errorf("parsing URL: %w", err) + } + request, err := http.NewRequest( verb, - oc.url(path, "").String(), + oc.url(parsedURL.Path, parsedURL.RawQuery).String(), bytes.NewBuffer(bodyBytes), ) if err != nil { @@ -206,14 +212,15 @@ func (oc *OrbitClient) ExecuteConfigReceivers() error { case <-oc.ReceiverUpdateContext.Done(): return nil case <-ticker.C: - err := oc.RunConfigReceivers() - log.Error().Err(err) + if err := oc.RunConfigReceivers(); err != nil { + log.Error().Err(err).Msg("running config receivers") + } } } } func (oc *OrbitClient) InterruptConfigReceivers(err error) { - log.Error().Err(err) + log.Error().Err(err).Msg("interrupt config receivers") oc.ReceiverUpdateCancelFunc() } @@ -313,6 +320,39 @@ func (oc *OrbitClient) SaveHostScriptResult(result *fleet.HostScriptResultPayloa return nil } +func (oc *OrbitClient) GetInstallerDetails(installId string) (*fleet.SoftwareInstallDetails, error) { + verb, path := "POST", "/api/fleet/orbit/software_install/details" + var resp orbitGetSoftwareInstallResponse + if err := oc.authenticatedRequest(verb, path, &orbitGetSoftwareInstallRequest{ + InstallUUID: installId, + }, &resp); err != nil { + return nil, err + } + return resp.SoftwareInstallDetails, nil +} + +func (oc *OrbitClient) SaveInstallerResult(payload *fleet.HostSoftwareInstallResultPayload) error { + verb, path := "POST", "/api/fleet/orbit/software_install/result" + var resp orbitPostSoftwareInstallResultResponse + if err := oc.authenticatedRequest(verb, path, &orbitPostSoftwareInstallResultRequest{ + HostSoftwareInstallResultPayload: payload, + }, &resp); err != nil { + return err + } + return nil +} + +func (oc *OrbitClient) DownloadSoftwareInstaller(installerID uint, downloadDirectory string) (string, error) { + verb, path := "POST", "/api/fleet/orbit/software_install/package?alt=media" + resp := FileResponse{DestPath: downloadDirectory} + if err := oc.authenticatedRequest(verb, path, &orbitDownloadSoftwareInstallerRequest{ + InstallerID: installerID, + }, &resp); err != nil { + return "", err + } + return resp.GetFilePath(), nil +} + // Ping sends a ping request to the orbit/ping endpoint. func (oc *OrbitClient) Ping() error { verb, path := "HEAD", "/api/fleet/orbit/ping" diff --git a/server/service/orbit_test.go b/server/service/orbit_test.go index f0cf515c64..e04ac966b9 100644 --- a/server/service/orbit_test.go +++ b/server/service/orbit_test.go @@ -32,6 +32,9 @@ func TestGetOrbitConfigNudge(t *testing.T) { ds.ListPendingHostScriptExecutionsFunc = func(ctx context.Context, hostID uint) ([]*fleet.HostScriptResult, error) { return nil, nil } + ds.ListPendingSoftwareInstallsFunc = func(ctx context.Context, hostID uint) ([]string, error) { + return nil, nil + } ctx = test.HostContext(ctx, &fleet.Host{ OsqueryHostID: ptr.String("test"), ID: 1, @@ -80,6 +83,9 @@ func TestGetOrbitConfigNudge(t *testing.T) { ds.GetHostOperatingSystemFunc = func(ctx context.Context, hostID uint) (*fleet.OperatingSystem, error) { return os, nil } + ds.ListPendingSoftwareInstallsFunc = func(ctx context.Context, hostID uint) ([]string, error) { + return nil, nil + } team := fleet.Team{ID: 1} teamMDM := fleet.TeamMDM{} ds.TeamMDMConfigFunc = func(ctx context.Context, teamID uint) (*fleet.TeamMDM, error) { @@ -163,7 +169,9 @@ func TestGetOrbitConfigNudge(t *testing.T) { ds.TeamAgentOptionsFunc = func(ctx context.Context, id uint) (*json.RawMessage, error) { return ptr.RawMessage(json.RawMessage(`{}`)), nil } - + ds.ListPendingSoftwareInstallsFunc = func(ctx context.Context, hostID uint) ([]string, error) { + return nil, nil + } checkEmptyNudgeConfig := func(h *fleet.Host) { ctx := test.HostContext(ctx, h) cfg, err := svc.GetOrbitConfig(ctx) @@ -248,7 +256,9 @@ func TestGetOrbitConfigNudge(t *testing.T) { ds.ListPendingHostScriptExecutionsFunc = func(ctx context.Context, hostID uint) ([]*fleet.HostScriptResult, error) { return nil, nil } - + ds.ListPendingSoftwareInstallsFunc = func(ctx context.Context, hostID uint) ([]string, error) { + return nil, nil + } appCfg := &fleet.AppConfig{MDM: fleet.MDM{EnabledAndConfigured: true}} appCfg.MDM.MacOSUpdates.Deadline = optjson.SetString("2022-04-01") appCfg.MDM.MacOSUpdates.MinimumVersion = optjson.SetString("12.3") diff --git a/server/service/software_installers.go b/server/service/software_installers.go index 4530d97c62..9a51b7f5f4 100644 --- a/server/service/software_installers.go +++ b/server/service/software_installers.go @@ -168,10 +168,10 @@ func getSoftwareInstallerEndpoint(ctx context.Context, request interface{}, svc payload, err := svc.DownloadSoftwareInstaller(ctx, req.TitleID, req.TeamID) if err != nil { - return downloadSoftwareInstallerResponse{Err: err}, nil + return orbitDownloadSoftwareInstallerResponse{Err: err}, nil } - return downloadSoftwareInstallerResponse{payload: payload}, nil + return orbitDownloadSoftwareInstallerResponse{payload: payload}, nil } func (svc *Service) GetSoftwareInstallerMetadata(ctx context.Context, titleID uint, teamID *uint) (*fleet.SoftwareInstaller, error) { @@ -182,15 +182,15 @@ func (svc *Service) GetSoftwareInstallerMetadata(ctx context.Context, titleID ui return nil, fleet.ErrMissingLicense } -type downloadSoftwareInstallerResponse struct { +type orbitDownloadSoftwareInstallerResponse struct { Err error `json:"error,omitempty"` // fields used by hijackRender for the response. payload *fleet.DownloadSoftwareInstallerPayload } -func (r downloadSoftwareInstallerResponse) error() error { return r.Err } +func (r orbitDownloadSoftwareInstallerResponse) error() error { return r.Err } -func (r downloadSoftwareInstallerResponse) hijackRender(ctx context.Context, w http.ResponseWriter) { +func (r orbitDownloadSoftwareInstallerResponse) hijackRender(ctx context.Context, w http.ResponseWriter) { w.Header().Set("Content-Length", strconv.Itoa(int(r.payload.Size))) w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment;filename="%s"`, r.payload.Filename)) From 1def5b2ddfef8b6ab14cba067607712d737d0f6c Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Tue, 14 May 2024 16:58:58 -0400 Subject: [PATCH 43/56] Add support for software installers in `fleetctl gitops` (#18990) --- .../18325-add-software-installers-to-fleetctl | 1 + cmd/fleetctl/gitops_test.go | 180 ++++++++++++++---- .../testdata/gitops/lib/install_ruby.sh | 1 + .../testdata/gitops/lib/post_install_ruby.sh | 1 + .../testdata/gitops/lib/query_multiple.yml | 11 ++ .../testdata/gitops/lib/query_ruby.yml | 5 + ...m_software_installer_install_not_found.yml | 18 ++ .../gitops/team_software_installer_no_url.yml | 21 ++ .../team_software_installer_not_found.yml | 16 ++ ...tware_installer_post_install_not_found.yml | 20 ++ ...staller_pre_condition_multiple_queries.yml | 22 +++ ...ware_installer_pre_condition_not_found.yml | 20 ++ .../team_software_installer_too_large.yml | 16 ++ .../team_software_installer_unsupported.yml | 16 ++ .../gitops/team_software_installer_valid.yml | 22 +++ ee/server/service/software_installers.go | 11 +- pkg/spec/gitops.go | 37 +++- pkg/spec/gitops_test.go | 6 +- server/service/client.go | 11 +- 19 files changed, 377 insertions(+), 58 deletions(-) create mode 100644 changes/18325-add-software-installers-to-fleetctl create mode 100644 cmd/fleetctl/testdata/gitops/lib/install_ruby.sh create mode 100644 cmd/fleetctl/testdata/gitops/lib/post_install_ruby.sh create mode 100644 cmd/fleetctl/testdata/gitops/lib/query_multiple.yml create mode 100644 cmd/fleetctl/testdata/gitops/lib/query_ruby.yml create mode 100644 cmd/fleetctl/testdata/gitops/team_software_installer_install_not_found.yml create mode 100644 cmd/fleetctl/testdata/gitops/team_software_installer_no_url.yml create mode 100644 cmd/fleetctl/testdata/gitops/team_software_installer_not_found.yml create mode 100644 cmd/fleetctl/testdata/gitops/team_software_installer_post_install_not_found.yml create mode 100644 cmd/fleetctl/testdata/gitops/team_software_installer_pre_condition_multiple_queries.yml create mode 100644 cmd/fleetctl/testdata/gitops/team_software_installer_pre_condition_not_found.yml create mode 100644 cmd/fleetctl/testdata/gitops/team_software_installer_too_large.yml create mode 100644 cmd/fleetctl/testdata/gitops/team_software_installer_unsupported.yml create mode 100644 cmd/fleetctl/testdata/gitops/team_software_installer_valid.yml diff --git a/changes/18325-add-software-installers-to-fleetctl b/changes/18325-add-software-installers-to-fleetctl new file mode 100644 index 0000000000..9171ee5219 --- /dev/null +++ b/changes/18325-add-software-installers-to-fleetctl @@ -0,0 +1 @@ +* Added `software` team setting to add software installers in YAML files for `fleetctl apply` and `fleetctl gitops`. diff --git a/cmd/fleetctl/gitops_test.go b/cmd/fleetctl/gitops_test.go index 5e87d4e2c2..36a354a994 100644 --- a/cmd/fleetctl/gitops_test.go +++ b/cmd/fleetctl/gitops_test.go @@ -3,7 +3,10 @@ package main import ( "context" "fmt" + "net/http" + "net/http/httptest" "os" + "path/filepath" "slices" "strings" "testing" @@ -20,7 +23,11 @@ import ( "github.com/stretchr/testify/require" ) -const teamName = "Team Test" +const ( + teamName = "Team Test" + fleetServerURL = "https://fleet.example.com" + orgName = "GitOps Test" +) func TestBasicGlobalGitOps(t *testing.T) { // Cannot run t.Parallel() because it sets environment variables @@ -196,6 +203,9 @@ func TestBasicTeamGitOps(t *testing.T) { ds.DeleteMDMAppleDeclarationByNameFunc = func(ctx context.Context, teamID *uint, name string) error { return nil } + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + return nil + } var enrolledSecrets []*fleet.EnrollSecret ds.ApplyEnrollSecretsFunc = func(ctx context.Context, teamID *uint, secrets []*fleet.EnrollSecret) error { @@ -549,6 +559,9 @@ func TestFullTeamGitOps(t *testing.T) { appliedQueries = queries return nil } + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + return nil + } var enrolledSecrets []*fleet.EnrollSecret ds.ApplyEnrollSecretsFunc = func(ctx context.Context, teamID *uint, secrets []*fleet.EnrollSecret) error { @@ -736,6 +749,9 @@ func TestBasicGlobalAndTeamGitOps(t *testing.T) { declaration.DeclarationUUID = uuid.NewString() return declaration, nil } + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + return nil + } globalFile, err := os.CreateTemp(t.TempDir(), "*.yml") require.NoError(t, err) @@ -850,6 +866,123 @@ team_settings: func TestFullGlobalAndTeamGitOps(t *testing.T) { // Cannot run t.Parallel() because it sets environment variables // mdm test configuration must be set so that activating windows MDM works. + ds, savedAppConfigPtr, savedTeamPtr := setupFullGitOpsPremiumServer(t) + + var enrolledSecrets []*fleet.EnrollSecret + var enrolledTeamSecrets []*fleet.EnrollSecret + var appliedPolicySpecs []*fleet.PolicySpec + var appliedQueries []*fleet.Query + + ds.ApplyEnrollSecretsFunc = func(ctx context.Context, teamID *uint, secrets []*fleet.EnrollSecret) error { + if teamID == nil { + enrolledSecrets = secrets + } else { + enrolledTeamSecrets = secrets + } + return nil + } + ds.ApplyPolicySpecsFunc = func(ctx context.Context, authorID uint, specs []*fleet.PolicySpec) error { + appliedPolicySpecs = specs + return nil + } + ds.ApplyQueriesFunc = func( + ctx context.Context, authorID uint, queries []*fleet.Query, queriesToDiscardResults map[uint]struct{}, + ) error { + appliedQueries = queries + return nil + } + ds.NewTeamFunc = func(ctx context.Context, team *fleet.Team) (*fleet.Team, error) { + team.ID = 1 + *savedTeamPtr = team + enrolledTeamSecrets = team.Secrets + return *savedTeamPtr, nil + } + + globalFile := "./testdata/gitops/global_config_no_paths.yml" + teamFile := "./testdata/gitops/team_config_no_paths.yml" + + // Dry run on global file should fail because Apple BM Default Team does not exist (and has not been provided) + _, err := runAppNoChecks([]string{"gitops", "-f", globalFile, "--dry-run"}) + require.Error(t, err) + assert.True(t, strings.Contains(err.Error(), "team name not found")) + + // Dry run + _ = runAppForTest(t, []string{"gitops", "-f", globalFile, "-f", teamFile, "--dry-run", "--delete-other-teams"}) + assert.False(t, ds.SaveAppConfigFuncInvoked) + assert.Len(t, enrolledSecrets, 0) + assert.Len(t, enrolledTeamSecrets, 0) + assert.Len(t, appliedPolicySpecs, 0) + assert.Len(t, appliedQueries, 0) + + // Real run + _ = runAppForTest(t, []string{"gitops", "-f", globalFile, "-f", teamFile, "--delete-other-teams"}) + assert.Equal(t, orgName, (*savedAppConfigPtr).OrgInfo.OrgName) + assert.Equal(t, fleetServerURL, (*savedAppConfigPtr).ServerSettings.ServerURL) + assert.Len(t, enrolledSecrets, 2) + require.NotNil(t, *savedTeamPtr) + assert.Equal(t, teamName, (*savedTeamPtr).Name) + require.Len(t, enrolledTeamSecrets, 2) +} + +func TestTeamSofwareInstallersGitOps(t *testing.T) { + // start the web server that will serve the installer + b, err := os.ReadFile(filepath.Join("..", "..", "server", "service", "testdata", "software-installers", "ruby.deb")) + require.NoError(t, err) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.Contains(r.URL.Path, "notfound"): + w.WriteHeader(http.StatusNotFound) + return + case strings.HasSuffix(r.URL.Path, ".txt"): + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte(`a simple text file`)) + return + case strings.Contains(r.URL.Path, "toolarge"): + w.Header().Set("Content-Type", "application/vnd.debian.binary-package") + var sz int + for sz < 500*1024*1024 { + n, _ := w.Write(b) + sz += n + } + default: + w.Header().Set("Content-Type", "application/vnd.debian.binary-package") + _, _ = w.Write(b) + } + })) + t.Cleanup(srv.Close) + t.Setenv("SOFTWARE_INSTALLER_URL", srv.URL) + + cases := []struct { + file string + wantErr string + }{ + {"testdata/gitops/team_software_installer_not_found.yml", "Please make sure that URLs are publicy accessible to the internet."}, + {"testdata/gitops/team_software_installer_unsupported.yml", "The file should be .pkg, .msi, .exe or .deb."}, + {"testdata/gitops/team_software_installer_too_large.yml", "The maximum file size is 500 MB"}, + {"testdata/gitops/team_software_installer_valid.yml", ""}, + {"testdata/gitops/team_software_installer_pre_condition_multiple_queries.yml", "should have only one query."}, + {"testdata/gitops/team_software_installer_pre_condition_not_found.yml", "no such file or directory"}, + {"testdata/gitops/team_software_installer_install_not_found.yml", "no such file or directory"}, + {"testdata/gitops/team_software_installer_post_install_not_found.yml", "no such file or directory"}, + {"testdata/gitops/team_software_installer_no_url.yml", "software URL is required"}, + } + for _, c := range cases { + t.Run(filepath.Base(c.file), func(t *testing.T) { + setupFullGitOpsPremiumServer(t) + + _, err := runAppNoChecks([]string{"gitops", "-f", c.file}) + if c.wantErr == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, c.wantErr) + } + }) + } + +} + +func setupFullGitOpsPremiumServer(t *testing.T) (*mock.Store, **fleet.AppConfig, **fleet.Team) { testCert, testKey, err := apple_mdm.NewSCEPCACertKey() require.NoError(t, err) testCertPEM := tokenpki.PEMCertificate(testCert.Raw) @@ -884,32 +1017,17 @@ func TestFullGlobalAndTeamGitOps(t *testing.T) { return nil } - const ( - fleetServerURL = "https://fleet.example.com" - orgName = "GitOps Test" - ) - var enrolledSecrets []*fleet.EnrollSecret - var enrolledTeamSecrets []*fleet.EnrollSecret - var appliedPolicySpecs []*fleet.PolicySpec - var appliedQueries []*fleet.Query var savedTeam *fleet.Team ds.ApplyEnrollSecretsFunc = func(ctx context.Context, teamID *uint, secrets []*fleet.EnrollSecret) error { - if teamID == nil { - enrolledSecrets = secrets - } else { - enrolledTeamSecrets = secrets - } return nil } ds.ApplyPolicySpecsFunc = func(ctx context.Context, authorID uint, specs []*fleet.PolicySpec) error { - appliedPolicySpecs = specs return nil } ds.ApplyQueriesFunc = func( ctx context.Context, authorID uint, queries []*fleet.Query, queriesToDiscardResults map[uint]struct{}, ) error { - appliedQueries = queries return nil } ds.BatchSetMDMProfilesFunc = func( @@ -957,7 +1075,6 @@ func TestFullGlobalAndTeamGitOps(t *testing.T) { ds.NewTeamFunc = func(ctx context.Context, team *fleet.Team) (*fleet.Team, error) { team.ID = 1 savedTeam = team - enrolledTeamSecrets = team.Secrets return savedTeam, nil } ds.QueryByNameFunc = func(ctx context.Context, teamID *uint, name string) (*fleet.Query, error) { @@ -985,35 +1102,14 @@ func TestFullGlobalAndTeamGitOps(t *testing.T) { declaration.DeclarationUUID = uuid.NewString() return declaration, nil } + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + return nil + } t.Setenv("FLEET_SERVER_URL", fleetServerURL) t.Setenv("ORG_NAME", orgName) t.Setenv("TEST_TEAM_NAME", teamName) t.Setenv("APPLE_BM_DEFAULT_TEAM", teamName) - globalFile := "./testdata/gitops/global_config_no_paths.yml" - teamFile := "./testdata/gitops/team_config_no_paths.yml" - - // Dry run on global file should fail because Apple BM Default Team does not exist (and has not been provided) - _, err = runAppNoChecks([]string{"gitops", "-f", globalFile, "--dry-run"}) - require.Error(t, err) - assert.True(t, strings.Contains(err.Error(), "team name not found")) - - // Dry run - _ = runAppForTest(t, []string{"gitops", "-f", globalFile, "-f", teamFile, "--dry-run", "--delete-other-teams"}) - assert.False(t, ds.SaveAppConfigFuncInvoked) - assert.Len(t, enrolledSecrets, 0) - assert.Len(t, enrolledTeamSecrets, 0) - assert.Len(t, appliedPolicySpecs, 0) - assert.Len(t, appliedQueries, 0) - - // Real run - _ = runAppForTest(t, []string{"gitops", "-f", globalFile, "-f", teamFile, "--delete-other-teams"}) - assert.Equal(t, orgName, savedAppConfig.OrgInfo.OrgName) - assert.Equal(t, fleetServerURL, savedAppConfig.ServerSettings.ServerURL) - assert.Len(t, enrolledSecrets, 2) - require.NotNil(t, savedTeam) - assert.Equal(t, teamName, savedTeam.Name) - require.Len(t, enrolledTeamSecrets, 2) - + return ds, &savedAppConfig, &savedTeam } diff --git a/cmd/fleetctl/testdata/gitops/lib/install_ruby.sh b/cmd/fleetctl/testdata/gitops/lib/install_ruby.sh new file mode 100644 index 0000000000..07f6784ab8 --- /dev/null +++ b/cmd/fleetctl/testdata/gitops/lib/install_ruby.sh @@ -0,0 +1 @@ +echo 'ruby' diff --git a/cmd/fleetctl/testdata/gitops/lib/post_install_ruby.sh b/cmd/fleetctl/testdata/gitops/lib/post_install_ruby.sh new file mode 100644 index 0000000000..4c24d26648 --- /dev/null +++ b/cmd/fleetctl/testdata/gitops/lib/post_install_ruby.sh @@ -0,0 +1 @@ +echo 'post ruby' diff --git a/cmd/fleetctl/testdata/gitops/lib/query_multiple.yml b/cmd/fleetctl/testdata/gitops/lib/query_multiple.yml new file mode 100644 index 0000000000..c3109b5f71 --- /dev/null +++ b/cmd/fleetctl/testdata/gitops/lib/query_multiple.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: query +spec: + name: query_ruby + query: select 1 +--- +apiVersion: v1 +kind: query +spec: + name: query_ruby2 + query: select 2 diff --git a/cmd/fleetctl/testdata/gitops/lib/query_ruby.yml b/cmd/fleetctl/testdata/gitops/lib/query_ruby.yml new file mode 100644 index 0000000000..28714447bf --- /dev/null +++ b/cmd/fleetctl/testdata/gitops/lib/query_ruby.yml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: query +spec: + name: query_ruby + query: select 1 diff --git a/cmd/fleetctl/testdata/gitops/team_software_installer_install_not_found.yml b/cmd/fleetctl/testdata/gitops/team_software_installer_install_not_found.yml new file mode 100644 index 0000000000..c3837b3a0e --- /dev/null +++ b/cmd/fleetctl/testdata/gitops/team_software_installer_install_not_found.yml @@ -0,0 +1,18 @@ +name: "${TEST_TEAM_NAME}" +team_settings: + secrets: + - secret: "ABC" + features: + enable_host_users: true + enable_software_inventory: true + host_expiry_settings: + host_expiry_enabled: true + host_expiry_window: 30 +agent_options: +controls: +policies: +queries: +software: + - url: ${SOFTWARE_INSTALLER_URL}/ruby.deb + install_script: + path: lib/notfound.sh diff --git a/cmd/fleetctl/testdata/gitops/team_software_installer_no_url.yml b/cmd/fleetctl/testdata/gitops/team_software_installer_no_url.yml new file mode 100644 index 0000000000..43bfa4babf --- /dev/null +++ b/cmd/fleetctl/testdata/gitops/team_software_installer_no_url.yml @@ -0,0 +1,21 @@ +name: "${TEST_TEAM_NAME}" +team_settings: + secrets: + - secret: "ABC" + features: + enable_host_users: true + enable_software_inventory: true + host_expiry_settings: + host_expiry_enabled: true + host_expiry_window: 30 +agent_options: +controls: +policies: +queries: +software: + - install_script: + path: lib/install_ruby.sh + pre_install_query: + path: lib/query_ruby.yml + post_install_script: + path: lib/post_install_ruby.sh diff --git a/cmd/fleetctl/testdata/gitops/team_software_installer_not_found.yml b/cmd/fleetctl/testdata/gitops/team_software_installer_not_found.yml new file mode 100644 index 0000000000..ca657a5736 --- /dev/null +++ b/cmd/fleetctl/testdata/gitops/team_software_installer_not_found.yml @@ -0,0 +1,16 @@ +name: "${TEST_TEAM_NAME}" +team_settings: + secrets: + - secret: "ABC" + features: + enable_host_users: true + enable_software_inventory: true + host_expiry_settings: + host_expiry_enabled: true + host_expiry_window: 30 +agent_options: +controls: +policies: +queries: +software: + - url: ${SOFTWARE_INSTALLER_URL}/notfound.deb diff --git a/cmd/fleetctl/testdata/gitops/team_software_installer_post_install_not_found.yml b/cmd/fleetctl/testdata/gitops/team_software_installer_post_install_not_found.yml new file mode 100644 index 0000000000..4cc9fbcef7 --- /dev/null +++ b/cmd/fleetctl/testdata/gitops/team_software_installer_post_install_not_found.yml @@ -0,0 +1,20 @@ +name: "${TEST_TEAM_NAME}" +team_settings: + secrets: + - secret: "ABC" + features: + enable_host_users: true + enable_software_inventory: true + host_expiry_settings: + host_expiry_enabled: true + host_expiry_window: 30 +agent_options: +controls: +policies: +queries: +software: + - url: ${SOFTWARE_INSTALLER_URL}/ruby.deb + install_script: + path: lib/install_ruby.sh + post_install_script: + path: lib/notfound.sh diff --git a/cmd/fleetctl/testdata/gitops/team_software_installer_pre_condition_multiple_queries.yml b/cmd/fleetctl/testdata/gitops/team_software_installer_pre_condition_multiple_queries.yml new file mode 100644 index 0000000000..4b26e63e4e --- /dev/null +++ b/cmd/fleetctl/testdata/gitops/team_software_installer_pre_condition_multiple_queries.yml @@ -0,0 +1,22 @@ +name: "${TEST_TEAM_NAME}" +team_settings: + secrets: + - secret: "ABC" + features: + enable_host_users: true + enable_software_inventory: true + host_expiry_settings: + host_expiry_enabled: true + host_expiry_window: 30 +agent_options: +controls: +policies: +queries: +software: + - url: ${SOFTWARE_INSTALLER_URL}/ruby.deb + install_script: + path: lib/install_ruby.sh + pre_install_query: + path: lib/query_multiple.yml + post_install_script: + path: lib/post_install_ruby.sh diff --git a/cmd/fleetctl/testdata/gitops/team_software_installer_pre_condition_not_found.yml b/cmd/fleetctl/testdata/gitops/team_software_installer_pre_condition_not_found.yml new file mode 100644 index 0000000000..681590d04d --- /dev/null +++ b/cmd/fleetctl/testdata/gitops/team_software_installer_pre_condition_not_found.yml @@ -0,0 +1,20 @@ +name: "${TEST_TEAM_NAME}" +team_settings: + secrets: + - secret: "ABC" + features: + enable_host_users: true + enable_software_inventory: true + host_expiry_settings: + host_expiry_enabled: true + host_expiry_window: 30 +agent_options: +controls: +policies: +queries: +software: + - url: ${SOFTWARE_INSTALLER_URL}/ruby.deb + install_script: + path: lib/install_ruby.sh + pre_install_query: + path: lib/notfound.yml diff --git a/cmd/fleetctl/testdata/gitops/team_software_installer_too_large.yml b/cmd/fleetctl/testdata/gitops/team_software_installer_too_large.yml new file mode 100644 index 0000000000..15d16e9e9a --- /dev/null +++ b/cmd/fleetctl/testdata/gitops/team_software_installer_too_large.yml @@ -0,0 +1,16 @@ +name: "${TEST_TEAM_NAME}" +team_settings: + secrets: + - secret: "ABC" + features: + enable_host_users: true + enable_software_inventory: true + host_expiry_settings: + host_expiry_enabled: true + host_expiry_window: 30 +agent_options: +controls: +policies: +queries: +software: + - url: ${SOFTWARE_INSTALLER_URL}/toolarge.deb diff --git a/cmd/fleetctl/testdata/gitops/team_software_installer_unsupported.yml b/cmd/fleetctl/testdata/gitops/team_software_installer_unsupported.yml new file mode 100644 index 0000000000..3f58009a03 --- /dev/null +++ b/cmd/fleetctl/testdata/gitops/team_software_installer_unsupported.yml @@ -0,0 +1,16 @@ +name: "${TEST_TEAM_NAME}" +team_settings: + secrets: + - secret: "ABC" + features: + enable_host_users: true + enable_software_inventory: true + host_expiry_settings: + host_expiry_enabled: true + host_expiry_window: 30 +agent_options: +controls: +policies: +queries: +software: + - url: ${SOFTWARE_INSTALLER_URL}/invalidtype.txt diff --git a/cmd/fleetctl/testdata/gitops/team_software_installer_valid.yml b/cmd/fleetctl/testdata/gitops/team_software_installer_valid.yml new file mode 100644 index 0000000000..42ec2fc59c --- /dev/null +++ b/cmd/fleetctl/testdata/gitops/team_software_installer_valid.yml @@ -0,0 +1,22 @@ +name: "${TEST_TEAM_NAME}" +team_settings: + secrets: + - secret: "ABC" + features: + enable_host_users: true + enable_software_inventory: true + host_expiry_settings: + host_expiry_enabled: true + host_expiry_window: 30 +agent_options: +controls: +policies: +queries: +software: + - url: ${SOFTWARE_INSTALLER_URL}/ruby.deb + install_script: + path: lib/install_ruby.sh + pre_install_query: + path: lib/query_ruby.yml + post_install_script: + path: lib/post_install_ruby.sh diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index a84617f10a..e8221d7488 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -319,7 +319,7 @@ func (svc *Service) addMetadataToSoftwarePayload(ctx context.Context, payload *f if err != nil { if errors.Is(err, file.ErrUnsupportedType) { return "", &fleet.BadRequestError{ - Message: "The file should be .pkg, .msi, .exe or .deb.", + Message: "Couldn't edit software. File type not supported. The file should be .pkg, .msi, .exe or .deb.", InternalErr: ctxerr.Wrap(ctx, err, "extracting metadata from installer"), } } @@ -429,6 +429,15 @@ func (svc *Service) BatchSetSoftwareInstallers(ctx context.Context, tmName strin bodyBytes, err := io.ReadAll(resp.Body) if err != nil { + // the max size error can be received either at client.Do or here when + // reading the body if it's caught via a limited body reader. + var maxBytesErr *http.MaxBytesError + if errors.Is(err, fleethttp.ErrMaxSizeExceeded) || errors.As(err, &maxBytesErr) { + return fleet.NewInvalidArgumentError( + "software.url", + fmt.Sprintf("Couldn't edit software. URL (%q). The maximum file size is %d MB", p.URL, maxInstallerSizeBytes/(1024*1024)), + ) + } return ctxerr.Wrapf(ctx, err, "reading installer %q contents", p.URL) } diff --git a/pkg/spec/gitops.go b/pkg/spec/gitops.go index fe5210caef..9e91adfbd5 100644 --- a/pkg/spec/gitops.go +++ b/pkg/spec/gitops.go @@ -4,14 +4,15 @@ import ( "encoding/json" "errors" "fmt" - "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/ghodss/yaml" - "github.com/hashicorp/go-multierror" - "golang.org/x/text/unicode/norm" "os" "path/filepath" "slices" "unicode" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/ghodss/yaml" + "github.com/hashicorp/go-multierror" + "golang.org/x/text/unicode/norm" ) type BaseItem struct { @@ -53,6 +54,8 @@ type GitOps struct { Controls Controls Policies []*fleet.PolicySpec Queries []*fleet.QuerySpec + // Software is only allowed on teams, not on global config. + Software []*fleet.TeamSpecSoftware } // GitOpsFromBytes parses a GitOps yaml file. @@ -66,7 +69,7 @@ func GitOpsFromBytes(b []byte, baseDir string) (*GitOps, error) { var multiError *multierror.Error result := &GitOps{} - topKeys := []string{"name", "team_settings", "org_settings", "agent_options", "controls", "policies", "queries"} + topKeys := []string{"name", "team_settings", "org_settings", "agent_options", "controls", "policies", "queries", "software"} for k := range top { if !slices.Contains(topKeys, k) { multiError = multierror.Append(multiError, fmt.Errorf("unknown top-level field: %s", k)) @@ -76,16 +79,18 @@ func GitOpsFromBytes(b []byte, baseDir string) (*GitOps, error) { // Figure out if this is an org or team settings file teamRaw, teamOk := top["name"] teamSettingsRaw, teamSettingsOk := top["team_settings"] + teamSoftware, teamSoftwareOk := top["software"] orgSettingsRaw, orgOk := top["org_settings"] if orgOk { - if teamOk || teamSettingsOk { - multiError = multierror.Append(multiError, errors.New("'org_settings' cannot be used with 'name' or 'team_settings'")) + if teamOk || teamSettingsOk || teamSoftwareOk { + multiError = multierror.Append(multiError, errors.New("'org_settings' cannot be used with 'name', 'team_settings' or 'software'")) } else { multiError = parseOrgSettings(orgSettingsRaw, result, baseDir, multiError) } } else if teamOk && teamSettingsOk { multiError = parseName(teamRaw, result, multiError) multiError = parseTeamSettings(teamSettingsRaw, result, baseDir, multiError) + multiError = parseSoftware(teamSoftware, result, baseDir, multiError) } else { multiError = multierror.Append(multiError, errors.New("either 'org_settings' or 'name' and 'team_settings' is required")) } @@ -456,6 +461,24 @@ func parseQueries(top map[string]json.RawMessage, result *GitOps, baseDir string return multiError } +func parseSoftware(softwareRaw json.RawMessage, result *GitOps, baseDir string, multiError *multierror.Error) *multierror.Error { + var softwareInstallers []fleet.TeamSpecSoftware + if len(softwareRaw) > 0 { + if err := json.Unmarshal(softwareRaw, &softwareInstallers); err != nil { + return multierror.Append(multiError, fmt.Errorf("failed to unmarshal software: %v", err)) + } + } + for _, item := range softwareInstallers { + item := item + if item.URL == "" { + multiError = multierror.Append(multiError, errors.New("software URL is required")) + continue + } + result.Software = append(result.Software, &item) + } + return multiError +} + func getDuplicateNames[T any](slice []T, getComparableString func(T) string) []string { // We are using the allKeys map as a set here. True means the item is a duplicate. allKeys := make(map[string]bool) diff --git a/pkg/spec/gitops_test.go b/pkg/spec/gitops_test.go index 98a0a36a0c..87aa46f0cc 100644 --- a/pkg/spec/gitops_test.go +++ b/pkg/spec/gitops_test.go @@ -234,20 +234,20 @@ func TestMixingGlobalAndTeamConfig(t *testing.T) { config := getGlobalConfig(nil) config += "name: TeamName\n" _, err := GitOpsFromBytes([]byte(config), "") - assert.ErrorContains(t, err, "'org_settings' cannot be used with 'name' or 'team_settings'") + assert.ErrorContains(t, err, "'org_settings' cannot be used with 'name', 'team_settings' or 'software'") // Mixing org_settings and team_settings config = getGlobalConfig(nil) config += "team_settings:\n secrets: []\n" _, err = GitOpsFromBytes([]byte(config), "") - assert.ErrorContains(t, err, "'org_settings' cannot be used with 'name' or 'team_settings'") + assert.ErrorContains(t, err, "'org_settings' cannot be used with 'name', 'team_settings' or 'software'") // Mixing org_settings and team name and team_settings config = getGlobalConfig(nil) config += "name: TeamName\n" config += "team_settings:\n secrets: []\n" _, err = GitOpsFromBytes([]byte(config), "") - assert.ErrorContains(t, err, "'org_settings' cannot be used with 'name' or 'team_settings'") + assert.ErrorContains(t, err, "'org_settings' cannot be used with 'name', 'team_settings' or 'software'") } func TestInvalidGitOpsYaml(t *testing.T) { diff --git a/server/service/client.go b/server/service/client.go index 4d6fe83d4a..3111f19d72 100644 --- a/server/service/client.go +++ b/server/service/client.go @@ -575,15 +575,15 @@ func (c *Client) ApplyGroup( group, err := spec.GroupFromBytes(rawSpec) if err != nil { - return nil, fmt.Errorf("unable to parse query spec file %s: %w", queryFile, err) + return nil, fmt.Errorf("Couldn't edit software (%s). Unable to parse pre-install query YAML file %s: %w", si.URL, queryFile, err) } if len(group.Queries) > 1 { - return nil, fmt.Errorf("pre_install_query file %s contains more than one query", queryFile) + return nil, fmt.Errorf("Couldn't edit software (%s). Pre-install query YAML file %s should have only one query.", si.URL, queryFile) } if len(group.Queries) == 0 { - return nil, fmt.Errorf("pre_install_query file %s doesn't have a query defined", queryFile) + return nil, fmt.Errorf("Couldn't edit software (%s). Pre-install query YAML file %s doesn't have a query defined.", si.URL, queryFile) } qc = group.Queries[0].Query @@ -594,7 +594,7 @@ func (c *Client) ApplyGroup( installScriptFile := resolveApplyRelativePath(baseDir, si.InstallScript.Path) ic, err = os.ReadFile(installScriptFile) if err != nil { - return nil, fmt.Errorf("applying fleet config: %w", err) + return nil, fmt.Errorf("Couldn't edit software (%s). Unable to read install script file %s: %w", si.URL, si.InstallScript.Path, err) } } @@ -603,7 +603,7 @@ func (c *Client) ApplyGroup( postInstallScriptFile := resolveApplyRelativePath(baseDir, si.PostInstallScript.Path) pc, err = os.ReadFile(postInstallScriptFile) if err != nil { - return nil, fmt.Errorf("applying fleet config: %w", err) + return nil, fmt.Errorf("Couldn't edit software (%s). Unable to read post-install script file %s: %w", si.URL, si.PostInstallScript.Path, err) } } @@ -1075,6 +1075,7 @@ func (c *Client) DoGitOps( team["features"] = features } team["scripts"] = scripts + team["software"] = config.Software team["secrets"] = config.TeamSettings["secrets"] team["webhook_settings"] = map[string]interface{}{} clearHostStatusWebhook := true From 79a121256e07ea3903225f2b7cb43205eefb3bd7 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 15 May 2024 08:40:06 -0400 Subject: [PATCH 44/56] Software installers: backend cleanup tasks part 2 (#18982) --- ee/server/service/software_installers.go | 24 +-- server/datastore/mysql/activities_test.go | 27 +-- server/datastore/mysql/hosts.go | 2 +- server/datastore/mysql/labels.go | 2 +- server/datastore/mysql/software_installers.go | 73 ++----- .../mysql/software_installers_test.go | 200 +++++++----------- .../datastore/mysql/software_titles_test.go | 2 +- server/fleet/datastore.go | 22 +- server/mock/datastore_mock.go | 34 +-- server/service/integration_core_test.go | 15 +- server/service/integration_enterprise_test.go | 196 ++++++++++------- server/service/software_installers_test.go | 2 +- server/service/software_titles.go | 2 +- 13 files changed, 263 insertions(+), 338 deletions(-) diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index e8221d7488..eb617ccffa 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -88,7 +88,7 @@ func (svc *Service) DeleteSoftwareInstaller(ctx context.Context, titleID uint, t return err } - meta, err := svc.ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, teamID, titleID) + meta, err := svc.ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, teamID, titleID, false) if err != nil { return ctxerr.Wrap(ctx, err, "getting software installer metadata") } @@ -128,7 +128,7 @@ func (svc *Service) GetSoftwareInstallerMetadata(ctx context.Context, titleID ui return nil, err } - meta, err := svc.ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, teamID, titleID) + meta, err := svc.ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, teamID, titleID, true) if err != nil { return nil, ctxerr.Wrap(ctx, err, "getting software installer metadata") } @@ -153,25 +153,21 @@ func (svc *Service) OrbitDownloadSoftwareInstaller(ctx context.Context, installe // this is not a user-authenticated endpoint svc.authz.SkipAuthorization(ctx) - host, ok := hostctx.FromContext(ctx) + _, ok := hostctx.FromContext(ctx) if !ok { return nil, fleet.OrbitError{Message: "internal error: missing host from request context"} } // get the installer's metadata - meta, err := svc.ds.GetSoftwareInstallerMetadata(ctx, installerID) + meta, err := svc.ds.GetSoftwareInstallerMetadataByID(ctx, installerID) if err != nil { return nil, ctxerr.Wrap(ctx, err, "getting software installer metadata") } - // ensure it cannot get access to a different team's installer - var hTeamID uint - if host.TeamID != nil { - hTeamID = *host.TeamID - } - if (meta.TeamID != nil && *meta.TeamID != hTeamID) || (meta.TeamID == nil && hTeamID != 0) { - return nil, ctxerr.Wrap(ctx, fleet.OrbitError{}, "host team does not match installer team") - } + // Note that we do allow downloading an installer that is on a different team + // than the host's team, because the install request might have come while + // the host was on that team, and then the host got moved to a different team + // but the request is still pending execution. return svc.getSoftwareInstallerBinary(ctx, meta.StorageID, meta.Name) } @@ -228,7 +224,7 @@ func (svc *Service) InstallSoftwareTitle(ctx context.Context, hostID uint, softw return err } - installer, err := svc.ds.GetSoftwareInstallerForTitle(ctx, softwareTitleID, host.TeamID) + installer, err := svc.ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, host.TeamID, softwareTitleID, false) if err != nil { if fleet.IsNotFound(err) { return &fleet.BadRequestError{ @@ -267,7 +263,7 @@ func (svc *Service) InstallSoftwareTitle(ctx context.Context, hostID uint, softw } } - err = svc.ds.InsertSoftwareInstallRequest(ctx, hostID, installer.InstallerID) + _, err = svc.ds.InsertSoftwareInstallRequest(ctx, hostID, installer.InstallerID) return ctxerr.Wrap(ctx, err, "inserting software install request") } diff --git a/server/datastore/mysql/activities_test.go b/server/datastore/mysql/activities_test.go index d2d252f473..5f4e9c03d5 100644 --- a/server/datastore/mysql/activities_test.go +++ b/server/datastore/mysql/activities_test.go @@ -352,19 +352,11 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { Version: "0.0.2", }) require.NoError(t, err) - sw1Meta, err := ds.GetSoftwareInstallerMetadata(ctx, sw1) + sw1Meta, err := ds.GetSoftwareInstallerMetadataByID(ctx, sw1) require.NoError(t, err) - sw2Meta, err := ds.GetSoftwareInstallerMetadata(ctx, sw2) + sw2Meta, err := ds.GetSoftwareInstallerMetadataByID(ctx, sw2) require.NoError(t, err) - latestSoftwareInstallerUUID := func() string { - var id string - ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { - return sqlx.GetContext(ctx, q, &id, `SELECT execution_id FROM host_software_installs ORDER BY id DESC LIMIT 1`) - }) - return id - } - // create some script requests for h1 hsr, err := ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h1.ID, ScriptID: &scr1.ID, ScriptContents: scr1.ScriptContents, UserID: &u.ID}) require.NoError(t, err) @@ -382,21 +374,18 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { require.NoError(t, err) h1E := hsr.ExecutionID // create some software installs requests for h1, make some complete - err = ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1Meta.InstallerID) + h1FooFailed, err := ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1Meta.InstallerID) require.NoError(t, err) - h1FooFailed := latestSoftwareInstallerUUID() - err = ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw2Meta.InstallerID) + h1Bar, err := ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw2Meta.InstallerID) require.NoError(t, err) - h1Bar := latestSoftwareInstallerUUID() err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ HostID: h1.ID, InstallUUID: h1FooFailed, PreInstallConditionOutput: ptr.String(""), // pre-install failed }) require.NoError(t, err) - err = ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1Meta.InstallerID) + h1FooInstalled, err := ds.InsertSoftwareInstallRequest(ctx, h1.ID, sw1Meta.InstallerID) require.NoError(t, err) - h1FooInstalled := latestSoftwareInstallerUUID() err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ HostID: h1.ID, InstallUUID: h1FooInstalled, @@ -404,9 +393,8 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { InstallScriptExitCode: ptr.Int(0), }) require.NoError(t, err) - err = ds.InsertSoftwareInstallRequest(noUserCtx, h1.ID, sw1Meta.InstallerID) // no user for this one + h1Foo, err := ds.InsertSoftwareInstallRequest(noUserCtx, h1.ID, sw1Meta.InstallerID) // no user for this one require.NoError(t, err) - h1Foo := latestSoftwareInstallerUUID() // create a single pending request for h2, as well as a non-pending one hsr, err = ds.NewHostScriptExecutionRequest(ctx, &fleet.HostScriptRequestPayload{HostID: h2.ID, ScriptID: &scr1.ID, ScriptContents: scr1.ScriptContents, UserID: &u.ID}) @@ -418,9 +406,8 @@ func testListHostUpcomingActivities(t *testing.T, ds *Datastore) { require.NoError(t, err) h2F := hsr.ExecutionID // add a pending software install request for h2 - err = ds.InsertSoftwareInstallRequest(ctx, h2.ID, sw2Meta.InstallerID) + h2Bar, err := ds.InsertSoftwareInstallRequest(ctx, h2.ID, sw2Meta.InstallerID) require.NoError(t, err) - h2Bar := latestSoftwareInstallerUUID() // nothing for h3 diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index 721750345a..b9c64045cd 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -1018,7 +1018,7 @@ func (ds *Datastore) applyHostFilters( // so we're reusing the same filter to avoid adding unnecessary conditions. if opt.SoftwareStatusFilter != nil { // get the installer id - meta, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, opt.TeamFilter, *opt.SoftwareTitleIDFilter) + meta, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, opt.TeamFilter, *opt.SoftwareTitleIDFilter, false) if err != nil { return "", nil, ctxerr.Wrap(ctx, err, "get software installer metadata by team and title id") } diff --git a/server/datastore/mysql/labels.go b/server/datastore/mysql/labels.go index e119ec3ec7..371d556b16 100644 --- a/server/datastore/mysql/labels.go +++ b/server/datastore/mysql/labels.go @@ -604,7 +604,7 @@ func (ds *Datastore) applyHostLabelFilters(ctx context.Context, filter fleet.Tea // } if opt.SoftwareTitleIDFilter != nil && opt.SoftwareStatusFilter != nil { // get the installer id - meta, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, opt.TeamFilter, *opt.SoftwareTitleIDFilter) + meta, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, opt.TeamFilter, *opt.SoftwareTitleIDFilter, false) if err != nil { return "", nil, ctxerr.Wrap(ctx, err, "get software installer metadata by team and title id") } diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 9e6bb10398..17588a13c4 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -150,7 +150,7 @@ func (ds *Datastore) getOrGenerateSoftwareInstallerTitleID(ctx context.Context, return titleID, nil } -func (ds *Datastore) GetSoftwareInstallerMetadata(ctx context.Context, id uint) (*fleet.SoftwareInstaller, error) { +func (ds *Datastore) GetSoftwareInstallerMetadataByID(ctx context.Context, id uint) (*fleet.SoftwareInstaller, error) { query := ` SELECT si.id, @@ -182,8 +182,15 @@ WHERE return &dest, nil } -func (ds *Datastore) GetSoftwareInstallerMetadataByTeamAndTitleID(ctx context.Context, teamID *uint, titleID uint) (*fleet.SoftwareInstaller, error) { - query := ` +func (ds *Datastore) GetSoftwareInstallerMetadataByTeamAndTitleID(ctx context.Context, teamID *uint, titleID uint, withScriptContents bool) (*fleet.SoftwareInstaller, error) { + var scriptContentsSelect, scriptContentsFrom string + if withScriptContents { + scriptContentsSelect = ` , inst.contents AS install_script, COALESCE(pisnt.contents, '') AS post_install_script ` + scriptContentsFrom = ` LEFT OUTER JOIN script_contents inst ON inst.id = si.install_script_content_id + LEFT OUTER JOIN script_contents pisnt ON pisnt.id = si.post_install_script_content_id ` + } + + query := fmt.Sprintf(` SELECT si.id, si.team_id, @@ -195,20 +202,15 @@ SELECT si.pre_install_query, si.post_install_script_content_id, si.uploaded_at, - inst.contents AS install_script, - COALESCE(pisnt.contents, '') AS post_install_script, COALESCE(st.name, '') AS software_title + %s FROM software_installers si LEFT OUTER JOIN software_titles st ON st.id = si.title_id - LEFT OUTER JOIN - script_contents inst - ON inst.id = si.install_script_content_id - LEFT OUTER JOIN - script_contents pisnt - ON pisnt.id = si.post_install_script_content_id + %s WHERE - si.title_id = ? AND si.global_or_team_id = ?` + si.title_id = ? AND si.global_or_team_id = ?`, + scriptContentsSelect, scriptContentsFrom) var tmID uint if teamID != nil { @@ -241,43 +243,7 @@ func (ds *Datastore) DeleteSoftwareInstaller(ctx context.Context, id uint) error return nil } -func (ds *Datastore) GetSoftwareInstallerForTitle(ctx context.Context, softwareTitleID uint, teamID *uint) (*fleet.SoftwareInstaller, error) { - var tmID uint - if teamID != nil { - tmID = *teamID - } - - const getInstallerIDStmt = ` -SELECT - id, - team_id, - title_id, - storage_id, - filename, - version, - install_script_content_id, - pre_install_query, - post_install_script_content_id, - uploaded_at -FROM - software_installers -WHERE - title_id = ? AND global_or_team_id = ?` - - var installer fleet.SoftwareInstaller - err := sqlx.GetContext(ctx, ds.reader(ctx), &installer, getInstallerIDStmt, softwareTitleID, tmID) - if err != nil { - if err == sql.ErrNoRows { - return nil, notFound("SoftwareInstaller") - } - - return nil, ctxerr.Wrap(ctx, err, "finding software installer by title") - } - - return &installer, nil -} - -func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareInstallerID uint) error { +func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareInstallerID uint) (string, error) { const ( insertStmt = ` INSERT INTO host_software_installs @@ -294,24 +260,25 @@ func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID ui err := sqlx.GetContext(ctx, ds.reader(ctx), &hostExists, hostExistsStmt, hostID) if err != nil { if err == sql.ErrNoRows { - return notFound("Host").WithID(hostID) + return "", notFound("Host").WithID(hostID) } - return ctxerr.Wrap(ctx, err, "checking if host exists") + return "", ctxerr.Wrap(ctx, err, "checking if host exists") } var userID *uint if ctxUser := authz.UserFromContext(ctx); ctxUser != nil { userID = &ctxUser.ID } + installID := uuid.NewString() _, err = ds.writer(ctx).ExecContext(ctx, insertStmt, - uuid.NewString(), + installID, hostID, softwareInstallerID, userID, ) - return ctxerr.Wrap(ctx, err, "inserting new install software request") + return installID, ctxerr.Wrap(ctx, err, "inserting new install software request") } func (ds *Datastore) GetSoftwareInstallResults(ctx context.Context, resultsUUID string) (*fleet.HostSoftwareInstallerResult, error) { diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go index 360ccfe98b..7eb58f8f65 100644 --- a/server/datastore/mysql/software_installers_test.go +++ b/server/datastore/mysql/software_installers_test.go @@ -3,19 +3,16 @@ package mysql import ( "bytes" "context" - "database/sql" "os" "path/filepath" "testing" "time" - "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/datastore/filesystem" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/ptr" "github.com/fleetdm/fleet/v4/server/test" "github.com/google/uuid" - "github.com/jmoiron/sqlx" "github.com/stretchr/testify/require" ) @@ -48,51 +45,60 @@ func testListPendingSoftwareInstalls(t *testing.T, ds *Datastore) { host1 := test.NewHost(t, ds, "host1", "1", "host1key", "host1uuid", time.Now()) host2 := test.NewHost(t, ds, "host2", "2", "host2key", "host2uuid", time.Now()) - script1, err := insertScriptContents(ctx, "hello", ds.writer(ctx)) - require.NoError(t, err) - script1Id, err := script1.LastInsertId() + installerID1, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "hello", + PreInstallQuery: "SELECT 1", + PostInstallScript: "world", + InstallerFile: bytes.NewReader([]byte("hello")), + StorageID: "storage1", + Filename: "file1", + Title: "file1", + Version: "1.0", + Source: "apps", + }) require.NoError(t, err) - script2, err := insertScriptContents(ctx, "world", ds.writer(ctx)) - require.NoError(t, err) - script2Id, err := script2.LastInsertId() + installerID2, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "world", + PreInstallQuery: "SELECT 2", + PostInstallScript: "hello", + InstallerFile: bytes.NewReader([]byte("hello")), + StorageID: "storage2", + Filename: "file2", + Title: "file2", + Version: "2.0", + Source: "apps", + }) require.NoError(t, err) - installer1, err := insertSoftwareInstaller(ctx, ds.writer(ctx), "file1", "1.0", "SELECT 1", "storage1", script1Id, script2Id) - require.NoError(t, err) - installer1Id, err := installer1.LastInsertId() + hostInstall1, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID1) require.NoError(t, err) - installer2, err := insertSoftwareInstaller(ctx, ds.writer(ctx), "file2", "2.0", "SELECT 2", "storage2", script2Id, script1Id) - require.NoError(t, err) - installer2Id, err := installer2.LastInsertId() + hostInstall2, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installerID2) require.NoError(t, err) - hostInstall1, err := insertHostSoftwareInstalls(ctx, ds.writer(ctx), host1.ID, "exec1", uint(installer1Id)) - require.NoError(t, err) - _ = hostInstall1 - - hostInstall2, err := insertHostSoftwareInstalls(ctx, ds.writer(ctx), host1.ID, "exec2", uint(installer2Id)) - require.NoError(t, err) - _ = hostInstall2 - - hostInstall3, err := insertHostSoftwareInstalls(ctx, ds.writer(ctx), host2.ID, "exec3", uint(installer1Id)) - require.NoError(t, err) - _ = hostInstall3 - - hostInstall4, err := insertHostSoftwareInstalls(ctx, ds.writer(ctx), host2.ID, "exec4", uint(installer2Id)) - require.NoError(t, err) - hostInstall4Id, err := hostInstall4.LastInsertId() + hostInstall3, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID1) require.NoError(t, err) - _ = ds.writer(ctx).MustExec("UPDATE host_software_installs SET install_script_exit_code = 0 WHERE id = ?", hostInstall4Id) - - hostInstall5, err := insertHostSoftwareInstalls(ctx, ds.writer(ctx), host2.ID, "exec5", uint(installer2Id)) - require.NoError(t, err) - hostInstall5Id, err := hostInstall5.LastInsertId() + hostInstall4, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID2) require.NoError(t, err) - _ = ds.writer(ctx).MustExec("UPDATE host_software_installs SET pre_install_query_output = 'output' WHERE id = ?", hostInstall5Id) + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: host2.ID, + InstallUUID: hostInstall4, + InstallScriptExitCode: ptr.Int(0), + }) + require.NoError(t, err) + + hostInstall5, err := ds.InsertSoftwareInstallRequest(ctx, host2.ID, installerID2) + require.NoError(t, err) + + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: host2.ID, + InstallUUID: hostInstall5, + PreInstallConditionOutput: ptr.String("output"), + }) + require.NoError(t, err) installDetailsList1, err := ds.ListPendingSoftwareInstalls(ctx, host1.ID) require.NoError(t, err) @@ -102,81 +108,22 @@ func testListPendingSoftwareInstalls(t *testing.T, ds *Datastore) { require.NoError(t, err) require.Equal(t, 1, len(installDetailsList2)) - require.Contains(t, installDetailsList1, "exec1") - require.Contains(t, installDetailsList1, "exec2") + require.Contains(t, installDetailsList1, hostInstall1) + require.Contains(t, installDetailsList1, hostInstall2) - require.Contains(t, installDetailsList2, "exec3") + require.Contains(t, installDetailsList2, hostInstall3) - exec1, err := ds.GetSoftwareInstallDetails(ctx, "exec1") + exec1, err := ds.GetSoftwareInstallDetails(ctx, hostInstall1) require.NoError(t, err) require.Equal(t, host1.ID, exec1.HostID) - require.Equal(t, "exec1", exec1.ExecutionID) + require.Equal(t, hostInstall1, exec1.ExecutionID) require.Equal(t, "hello", exec1.InstallScript) require.Equal(t, "world", exec1.PostInstallScript) - require.Equal(t, uint(installer1Id), exec1.InstallerID) + require.Equal(t, installerID1, exec1.InstallerID) require.Equal(t, "SELECT 1", exec1.PreInstallCondition) } -func insertHostSoftwareInstalls( - ctx context.Context, - tx sqlx.ExtContext, - hostId uint, - executionId string, - softwareInstallerId uint, -) (sql.Result, error) { - stmt := ` - INSERT INTO host_software_installs ( - host_id, - execution_id, - software_installer_id - ) VALUES (?, ?, ?) -` - res, err := tx.ExecContext(ctx, stmt, hostId, executionId, softwareInstallerId) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "inserting host software install") - } - - return res, nil -} - -func insertSoftwareInstaller( - ctx context.Context, - tx sqlx.ExtContext, - filename, - version, - preinstallQuery, - storageId string, - installScriptId, - postInstallScriptId int64, -) (sql.Result, error) { - stmt := ` - INSERT INTO software_installers ( - filename, - version, - pre_install_query, - install_script_content_id, - post_install_script_content_id, - storage_id - ) - VALUES (?, ?, ?, ?, ?, ?) -` - res, err := tx.ExecContext(ctx, - stmt, - filename, - version, - preinstallQuery, - installScriptId, - postInstallScriptId, - storageId, - ) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "inserting software installer") - } - - return res, nil -} - func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { ctx := context.Background() @@ -192,7 +139,7 @@ func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { for tc, teamID := range cases { t.Run(tc, func(t *testing.T) { // non-existent installer - si, err := ds.GetSoftwareInstallerForTitle(ctx, 1, teamID) + si, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, teamID, 1, false) var nfe fleet.NotFoundError require.ErrorAs(t, err, &nfe) require.Nil(t, si) @@ -205,16 +152,16 @@ func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { Filename: "foo.pkg", }) require.NoError(t, err) - installerMeta, err := ds.GetSoftwareInstallerMetadata(ctx, installerID) + installerMeta, err := ds.GetSoftwareInstallerMetadataByID(ctx, installerID) require.NoError(t, err) - si, err = ds.GetSoftwareInstallerForTitle(ctx, *installerMeta.TitleID, teamID) + si, err = ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, teamID, *installerMeta.TitleID, false) require.NoError(t, err) require.NotNil(t, si) require.Equal(t, "foo.pkg", si.Name) // non-existent host - err = ds.InsertSoftwareInstallRequest(ctx, 12, si.InstallerID) + _, err = ds.InsertSoftwareInstallRequest(ctx, 12, si.InstallerID) require.ErrorAs(t, err, &nfe) // successful insert @@ -227,7 +174,7 @@ func testSoftwareInstallRequests(t *testing.T, ds *Datastore) { TeamID: teamID, }) require.NoError(t, err) - err = ds.InsertSoftwareInstallRequest(ctx, host.ID, si.InstallerID) + _, err = ds.InsertSoftwareInstallRequest(ctx, host.ID, si.InstallerID) require.NoError(t, err) // list hosts with software install requests @@ -266,40 +213,35 @@ func testGetSoftwareInstallResult(t *testing.T, ds *Datastore) { for _, tc := range []struct { name string - uuid string expectedStatus fleet.SoftwareInstallerStatus - postInstallScriptEC *uint + postInstallScriptEC *int preInstallQueryOutput *string - installScriptEC *uint + installScriptEC *int postInstallScriptOutput *string installScriptOutput *string }{ { name: "pending install", - uuid: "pending", expectedStatus: fleet.SoftwareInstallerPending, postInstallScriptOutput: ptr.String("post install output"), installScriptOutput: ptr.String("install output"), }, { name: "failing install post install script", - uuid: "fail_post_install_script", expectedStatus: fleet.SoftwareInstallerFailed, - postInstallScriptEC: ptr.Uint(1), + postInstallScriptEC: ptr.Int(1), postInstallScriptOutput: ptr.String("post install output"), installScriptOutput: ptr.String("install output"), }, { name: "failing install install script", - uuid: "fail_install_script", expectedStatus: fleet.SoftwareInstallerFailed, - installScriptEC: ptr.Uint(1), + installScriptEC: ptr.Int(1), postInstallScriptOutput: ptr.String("post install output"), installScriptOutput: ptr.String("install output"), }, { name: "failing install pre install query", - uuid: "fail_pre_install_query", expectedStatus: fleet.SoftwareInstallerFailed, preInstallQueryOutput: ptr.String(""), postInstallScriptOutput: ptr.String("post install output"), @@ -328,15 +270,23 @@ func testGetSoftwareInstallResult(t *testing.T, ds *Datastore) { }) require.NoError(t, err) - // Need to insert manually so we have access to the UUID (it's generated in the DS method) - query := `INSERT INTO host_software_installs (execution_id, host_id, software_installer_id, post_install_script_exit_code, install_script_exit_code, pre_install_query_output, install_script_output, post_install_script_output) VALUES (?,?,?,?,?,?,?,?)` - _, err = ds.writer(ctx).ExecContext(ctx, query, tc.uuid, host.ID, installerID, tc.postInstallScriptEC, tc.installScriptEC, tc.preInstallQueryOutput, tc.installScriptOutput, tc.postInstallScriptOutput) + installUUID, err := ds.InsertSoftwareInstallRequest(ctx, host.ID, installerID) + require.NoError(t, err) + err = ds.SetHostSoftwareInstallResult(ctx, &fleet.HostSoftwareInstallResultPayload{ + HostID: host.ID, + InstallUUID: installUUID, + PreInstallConditionOutput: tc.preInstallQueryOutput, + InstallScriptExitCode: tc.installScriptEC, + InstallScriptOutput: tc.installScriptOutput, + PostInstallScriptExitCode: tc.postInstallScriptEC, + PostInstallScriptOutput: tc.postInstallScriptOutput, + }) require.NoError(t, err) - res, err := ds.GetSoftwareInstallResults(ctx, tc.uuid) + res, err := ds.GetSoftwareInstallResults(ctx, installUUID) require.NoError(t, err) - require.Equal(t, tc.uuid, res.InstallUUID) + require.Equal(t, installUUID, res.InstallUUID) require.Equal(t, tc.expectedStatus, res.Status) require.Equal(t, swFilename, res.SoftwarePackage) require.Equal(t, host.ID, res.HostID) @@ -436,7 +386,7 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { require.Len(t, titles, len(wantTitles)) for _, title := range titles { - meta, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, &team.ID, title.ID) + meta, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, &team.ID, title.ID, false) require.NoError(t, err) require.NotNil(t, meta.TitleID) } @@ -540,10 +490,10 @@ func testGetSoftwareInstallerMetadataByTeamAndTitleID(t *testing.T, ds *Datastor Filename: "foo.pkg", }) require.NoError(t, err) - installerMeta, err := ds.GetSoftwareInstallerMetadata(ctx, installerID) + installerMeta, err := ds.GetSoftwareInstallerMetadataByID(ctx, installerID) require.NoError(t, err) - metaByTeamAndTitle, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, &team.ID, *installerMeta.TitleID) + metaByTeamAndTitle, err := ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, &team.ID, *installerMeta.TitleID, true) require.NoError(t, err) require.Equal(t, "echo install", metaByTeamAndTitle.InstallScript) require.Equal(t, "echo post-install", metaByTeamAndTitle.PostInstallScript) @@ -558,10 +508,10 @@ func testGetSoftwareInstallerMetadataByTeamAndTitleID(t *testing.T, ds *Datastor Filename: "foo.pkg", }) require.NoError(t, err) - installerMeta, err = ds.GetSoftwareInstallerMetadata(ctx, installerID) + installerMeta, err = ds.GetSoftwareInstallerMetadataByID(ctx, installerID) require.NoError(t, err) - metaByTeamAndTitle, err = ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, &team.ID, *installerMeta.TitleID) + metaByTeamAndTitle, err = ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, &team.ID, *installerMeta.TitleID, true) require.NoError(t, err) require.Equal(t, "echo install", metaByTeamAndTitle.InstallScript) require.Equal(t, "", metaByTeamAndTitle.PostInstallScript) diff --git a/server/datastore/mysql/software_titles_test.go b/server/datastore/mysql/software_titles_test.go index c4dffd9a63..a6eb1a7b67 100644 --- a/server/datastore/mysql/software_titles_test.go +++ b/server/datastore/mysql/software_titles_test.go @@ -305,7 +305,7 @@ func testOrderSoftwareTitles(t *testing.T, ds *Datastore) { Filename: "installer2.pkg", }) require.NoError(t, err) - err = ds.InsertSoftwareInstallRequest(ctx, host1.ID, installer2) + _, err = ds.InsertSoftwareInstallRequest(ctx, host1.ID, installer2) require.NoError(t, err) require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now())) diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 71d7e2605f..76aaa3a3fa 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -490,12 +490,10 @@ type Datastore interface { ListSoftwareTitles(ctx context.Context, opt SoftwareTitleListOptions, tmFilter TeamFilter) ([]SoftwareTitle, int, *PaginationMetadata, error) SoftwareTitleByID(ctx context.Context, id uint, teamID *uint, tmFilter TeamFilter) (*SoftwareTitle, error) - // InsertSoftwareInstallRequest tracks a new request to install the provided software installer in the host - InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareTitleID uint) error - - // GetSoftwareInstallerForTitle returns the software installer - // associated with the given title - team combination. - GetSoftwareInstallerForTitle(ctx context.Context, softwareTitleID uint, teamID *uint) (*SoftwareInstaller, error) + // InsertSoftwareInstallRequest tracks a new request to install the provided + // software installer in the host. It returns the auto-generated installation + // uuid. + InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareTitleID uint) (string, error) /////////////////////////////////////////////////////////////////////////////// // SoftwareStore @@ -1485,12 +1483,14 @@ type Datastore interface { // MatchOrCreateSoftwareInstaller matches or creates a new software installer. MatchOrCreateSoftwareInstaller(ctx context.Context, payload *UploadSoftwareInstallerPayload) (uint, error) - // GetSoftwareInstallerMetadata returns the software installer corresponding to the installer id. - GetSoftwareInstallerMetadata(ctx context.Context, id uint) (*SoftwareInstaller, error) + // GetSoftwareInstallerMetadataByID returns the software installer corresponding to the installer id. + GetSoftwareInstallerMetadataByID(ctx context.Context, id uint) (*SoftwareInstaller, error) - // GetSoftwareInstallerMetadataByTitleID returns the software installer corresponding to the specified - // team and title ids. - GetSoftwareInstallerMetadataByTeamAndTitleID(ctx context.Context, teamID *uint, titleID uint) (*SoftwareInstaller, error) + // GetSoftwareInstallerMetadataByTitleID returns the software installer + // corresponding to the specified team and title ids. If withScriptContents + // is true, also returns the contents of the install and (if set) + // post-install scripts, otherwise those fields are left empty. + GetSoftwareInstallerMetadataByTeamAndTitleID(ctx context.Context, teamID *uint, titleID uint, withScriptContents bool) (*SoftwareInstaller, error) // DeleteSoftwareInstaller deletes the software installer corresponding to the id. DeleteSoftwareInstaller(ctx context.Context, id uint) error diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 923e6035f0..be3e08a7c3 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -365,9 +365,7 @@ type ListSoftwareTitlesFunc func(ctx context.Context, opt fleet.SoftwareTitleLis type SoftwareTitleByIDFunc func(ctx context.Context, id uint, teamID *uint, tmFilter fleet.TeamFilter) (*fleet.SoftwareTitle, error) -type InsertSoftwareInstallRequestFunc func(ctx context.Context, hostID uint, softwareTitleID uint) error - -type GetSoftwareInstallerForTitleFunc func(ctx context.Context, softwareTitleID uint, teamID *uint) (*fleet.SoftwareInstaller, error) +type InsertSoftwareInstallRequestFunc func(ctx context.Context, hostID uint, softwareTitleID uint) (string, error) type ListSoftwareForVulnDetectionFunc func(ctx context.Context, hostID uint) ([]fleet.Software, error) @@ -939,9 +937,9 @@ type ListPendingSoftwareInstallsFunc func(ctx context.Context, hostID uint) ([]s type MatchOrCreateSoftwareInstallerFunc func(ctx context.Context, payload *fleet.UploadSoftwareInstallerPayload) (uint, error) -type GetSoftwareInstallerMetadataFunc func(ctx context.Context, id uint) (*fleet.SoftwareInstaller, error) +type GetSoftwareInstallerMetadataByIDFunc func(ctx context.Context, id uint) (*fleet.SoftwareInstaller, error) -type GetSoftwareInstallerMetadataByTeamAndTitleIDFunc func(ctx context.Context, teamID *uint, titleID uint) (*fleet.SoftwareInstaller, error) +type GetSoftwareInstallerMetadataByTeamAndTitleIDFunc func(ctx context.Context, teamID *uint, titleID uint, withScriptContents bool) (*fleet.SoftwareInstaller, error) type DeleteSoftwareInstallerFunc func(ctx context.Context, id uint) error @@ -1476,9 +1474,6 @@ type DataStore struct { InsertSoftwareInstallRequestFunc InsertSoftwareInstallRequestFunc InsertSoftwareInstallRequestFuncInvoked bool - GetSoftwareInstallerForTitleFunc GetSoftwareInstallerForTitleFunc - GetSoftwareInstallerForTitleFuncInvoked bool - ListSoftwareForVulnDetectionFunc ListSoftwareForVulnDetectionFunc ListSoftwareForVulnDetectionFuncInvoked bool @@ -2334,8 +2329,8 @@ type DataStore struct { MatchOrCreateSoftwareInstallerFunc MatchOrCreateSoftwareInstallerFunc MatchOrCreateSoftwareInstallerFuncInvoked bool - GetSoftwareInstallerMetadataFunc GetSoftwareInstallerMetadataFunc - GetSoftwareInstallerMetadataFuncInvoked bool + GetSoftwareInstallerMetadataByIDFunc GetSoftwareInstallerMetadataByIDFunc + GetSoftwareInstallerMetadataByIDFuncInvoked bool GetSoftwareInstallerMetadataByTeamAndTitleIDFunc GetSoftwareInstallerMetadataByTeamAndTitleIDFunc GetSoftwareInstallerMetadataByTeamAndTitleIDFuncInvoked bool @@ -3569,20 +3564,13 @@ func (s *DataStore) SoftwareTitleByID(ctx context.Context, id uint, teamID *uint return s.SoftwareTitleByIDFunc(ctx, id, teamID, tmFilter) } -func (s *DataStore) InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareTitleID uint) error { +func (s *DataStore) InsertSoftwareInstallRequest(ctx context.Context, hostID uint, softwareTitleID uint) (string, error) { s.mu.Lock() s.InsertSoftwareInstallRequestFuncInvoked = true s.mu.Unlock() return s.InsertSoftwareInstallRequestFunc(ctx, hostID, softwareTitleID) } -func (s *DataStore) GetSoftwareInstallerForTitle(ctx context.Context, softwareTitleID uint, teamID *uint) (*fleet.SoftwareInstaller, error) { - s.mu.Lock() - s.GetSoftwareInstallerForTitleFuncInvoked = true - s.mu.Unlock() - return s.GetSoftwareInstallerForTitleFunc(ctx, softwareTitleID, teamID) -} - func (s *DataStore) ListSoftwareForVulnDetection(ctx context.Context, hostID uint) ([]fleet.Software, error) { s.mu.Lock() s.ListSoftwareForVulnDetectionFuncInvoked = true @@ -5578,18 +5566,18 @@ func (s *DataStore) MatchOrCreateSoftwareInstaller(ctx context.Context, payload return s.MatchOrCreateSoftwareInstallerFunc(ctx, payload) } -func (s *DataStore) GetSoftwareInstallerMetadata(ctx context.Context, id uint) (*fleet.SoftwareInstaller, error) { +func (s *DataStore) GetSoftwareInstallerMetadataByID(ctx context.Context, id uint) (*fleet.SoftwareInstaller, error) { s.mu.Lock() - s.GetSoftwareInstallerMetadataFuncInvoked = true + s.GetSoftwareInstallerMetadataByIDFuncInvoked = true s.mu.Unlock() - return s.GetSoftwareInstallerMetadataFunc(ctx, id) + return s.GetSoftwareInstallerMetadataByIDFunc(ctx, id) } -func (s *DataStore) GetSoftwareInstallerMetadataByTeamAndTitleID(ctx context.Context, teamID *uint, titleID uint) (*fleet.SoftwareInstaller, error) { +func (s *DataStore) GetSoftwareInstallerMetadataByTeamAndTitleID(ctx context.Context, teamID *uint, titleID uint, withScriptContents bool) (*fleet.SoftwareInstaller, error) { s.mu.Lock() s.GetSoftwareInstallerMetadataByTeamAndTitleIDFuncInvoked = true s.mu.Unlock() - return s.GetSoftwareInstallerMetadataByTeamAndTitleIDFunc(ctx, teamID, titleID) + return s.GetSoftwareInstallerMetadataByTeamAndTitleIDFunc(ctx, teamID, titleID, withScriptContents) } func (s *DataStore) DeleteSoftwareInstaller(ctx context.Context, id uint) error { diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index 0a884fdb05..ad70209431 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -8968,7 +8968,7 @@ func setOrbitEnrollment(t *testing.T, h *fleet.Host, ds fleet.Datastore) string _, err := ds.EnrollOrbit(context.Background(), false, fleet.OrbitHostInfo{ HardwareUUID: *h.OsqueryHostID, HardwareSerial: h.HardwareSerial, - }, orbitKey, nil) + }, orbitKey, h.TeamID) require.NoError(t, err) err = ds.SetOrUpdateHostOrbitInfo( context.Background(), h.ID, "1.22.0", sql.NullString{String: "42", Valid: true}, sql.NullBool{Bool: true, Valid: true}, @@ -11105,14 +11105,6 @@ func (s *integrationTestSuite) TestListHostUpcomingActivities() { // verifying that the service layer passes the proper options and the // rendering of the response. - latestSoftwareInstallerUUID := func() string { - var id string - mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { - return sqlx.GetContext(ctx, q, &id, `SELECT execution_id FROM host_software_installs ORDER BY id DESC LIMIT 1`) - }) - return id - } - host1, err := s.ds.NewHost(ctx, &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -11154,11 +11146,10 @@ func (s *integrationTestSuite) TestListHostUpcomingActivities() { Version: "0.0.1", }) require.NoError(t, err) - s1Meta, err := s.ds.GetSoftwareInstallerMetadata(ctx, sw1) + s1Meta, err := s.ds.GetSoftwareInstallerMetadataByID(ctx, sw1) require.NoError(t, err) - err = s.ds.InsertSoftwareInstallRequest(ctx, host1.ID, s1Meta.InstallerID) + h1Foo, err := s.ds.InsertSoftwareInstallRequest(ctx, host1.ID, s1Meta.InstallerID) require.NoError(t, err) - h1Foo := latestSoftwareInstallerUUID() // force an order to the activities endTime := mysql.SetOrderedCreatedAtTimestamps(t, s.ds, time.Now(), "host_script_results", "execution_id", h1A, h1B) diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index a431e2be76..afa6454db2 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -8864,7 +8864,7 @@ func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerUploadDownloadAndD }) require.NotZero(t, id) - meta, err := s.ds.GetSoftwareInstallerMetadata(context.Background(), id) + meta, err := s.ds.GetSoftwareInstallerMetadataByID(context.Background(), id) require.NoError(t, err) if payload.TeamID != nil { @@ -8916,6 +8916,12 @@ func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerUploadDownloadAndD // upload again fails s.uploadSoftwareInstaller(payload, http.StatusConflict, "already exists") + // orbit-downloading fails with invalid orbit node key + s.Do("POST", "/api/fleet/orbit/software_install/package?alt=media", orbitDownloadSoftwareInstallerRequest{ + InstallerID: 123, + OrbitNodeKey: uuid.NewString(), + }, http.StatusUnauthorized) + // download the installer s.Do("GET", fmt.Sprintf("/api/latest/fleet/software/%d/package?alt=media", titleID), nil, http.StatusBadRequest) @@ -8942,7 +8948,6 @@ func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerUploadDownloadAndD Source: "deb_packages", StorageID: "df06d9ce9e2090d9cb2e8cd1f4d7754a803dc452bf93e3204e3acd3b95508628", } - s.uploadSoftwareInstaller(payload, http.StatusOK, "") // check the software installer @@ -8958,12 +8963,30 @@ func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerUploadDownloadAndD r := s.Do("GET", fmt.Sprintf("/api/latest/fleet/software/%d/package?alt=media", titleID), nil, http.StatusOK, "team_id", fmt.Sprintf("%d", *payload.TeamID)) checkDownloadResponse(t, r, payload.Filename) - // create an orbit host, assign to team and request to download the installer - host := createOrbitEnrolledHost(t, "windows", "orbit-host-team", s.ds) - require.NoError(t, s.ds.AddHostsToTeam(context.Background(), &createTeamResp.Team.ID, []uint{host.ID})) + // create an orbit host that is not in the team + hostNotInTeam := createOrbitEnrolledHost(t, "windows", "orbit-host-no-team", s.ds) + // downloading installer still works because we allow it explicitly + s.Do("POST", "/api/fleet/orbit/software_install/package?alt=media", orbitDownloadSoftwareInstallerRequest{ + InstallerID: installerID, + OrbitNodeKey: *hostNotInTeam.OrbitNodeKey, + }, http.StatusOK) + + // create an orbit host, assign to team + hostInTeam := createOrbitEnrolledHost(t, "windows", "orbit-host-team", s.ds) + require.NoError(t, s.ds.AddHostsToTeam(context.Background(), &createTeamResp.Team.ID, []uint{hostInTeam.ID})) + + // requesting download with alt != media fails + r = s.Do("POST", "/api/fleet/orbit/software_install/package?alt=FOOBAR", orbitDownloadSoftwareInstallerRequest{ + InstallerID: installerID, + OrbitNodeKey: *hostInTeam.OrbitNodeKey, + }, http.StatusBadRequest) + errMsg := extractServerErrorText(r.Body) + require.Contains(t, errMsg, "only alt=media is supported") + + // valid download r = s.Do("POST", "/api/fleet/orbit/software_install/package?alt=media", orbitDownloadSoftwareInstallerRequest{ InstallerID: installerID, - OrbitNodeKey: *host.OrbitNodeKey, + OrbitNodeKey: *hostInTeam.OrbitNodeKey, }, http.StatusOK) checkDownloadResponse(t, r, payload.Filename) @@ -9287,6 +9310,7 @@ func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerHostRequests() { require.NoError(t, err) err = s.ds.AddHostsToTeam(context.Background(), teamID, []uint{h.ID}) require.NoError(t, err) + h.TeamID = teamID // request fails resp = installSoftwareResponse{} @@ -9294,11 +9318,6 @@ func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerHostRequests() { // host installs fleetd setOrbitEnrollment(t, h, s.ds) - // TODO(roberto) setOrbitEnrollment is a helper function that silently - // sets the team_id to NULL. We need to refactor it to accept a - // parameter with an optional team value. - err = s.ds.AddHostsToTeam(context.Background(), teamID, []uint{h.ID}) - require.NoError(t, err) // request fails because of non-existent title resp = installSoftwareResponse{} @@ -9324,6 +9343,8 @@ func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerHostRequests() { s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", h.ID), nil, http.StatusOK, &getHostSoftwareResp) require.Len(t, getHostSoftwareResp.Software, 1) require.NotNil(t, getHostSoftwareResp.Software[0].LastInstall) + require.NotNil(t, getHostSoftwareResp.Software[0].Status) + require.Equal(t, fleet.SoftwareInstallerPending, *getHostSoftwareResp.Software[0].Status) installUUID := getHostSoftwareResp.Software[0].LastInstall.InstallUUID gsirr := getSoftwareInstallResultsResponse{} @@ -9334,6 +9355,54 @@ func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerHostRequests() { require.Equal(t, installUUID, results.InstallUUID) require.Equal(t, fleet.SoftwareInstallerPending, results.Status) + // create 3 more hosts, will have statuses installed, failed and one with two + // install requests - one failed and the latest install pending + h2 := createOrbitEnrolledHost(t, "linux", "host2", s.ds) + h3 := createOrbitEnrolledHost(t, "linux", "host3", s.ds) + h4 := createOrbitEnrolledHost(t, "linux", "host4", s.ds) + err = s.ds.AddHostsToTeam(context.Background(), teamID, []uint{h2.ID, h3.ID, h4.ID}) + require.NoError(t, err) + + s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/install/%d", h2.ID, titleID), nil, http.StatusAccepted, &resp) + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", h2.ID), nil, http.StatusOK, &getHostSoftwareResp) + require.Len(t, getHostSoftwareResp.Software, 1) + installUUID2 := getHostSoftwareResp.Software[0].LastInstall.InstallUUID + s.Do("POST", "/api/fleet/orbit/software_install/result", json.RawMessage(fmt.Sprintf(`{ + "orbit_node_key": %q, + "install_uuid": %q, + "pre_install_condition_output": "ok", + "install_script_exit_code": 0, + "install_script_output": "ok" + }`, *h2.OrbitNodeKey, installUUID2)), http.StatusNoContent) + + s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/install/%d", h3.ID, titleID), nil, http.StatusAccepted, &resp) + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", h3.ID), nil, http.StatusOK, &getHostSoftwareResp) + require.Len(t, getHostSoftwareResp.Software, 1) + installUUID3 := getHostSoftwareResp.Software[0].LastInstall.InstallUUID + s.Do("POST", "/api/fleet/orbit/software_install/result", json.RawMessage(fmt.Sprintf(`{ + "orbit_node_key": %q, + "install_uuid": %q, + "pre_install_condition_output": "ok", + "install_script_exit_code": 1, + "install_script_output": "failed" + }`, *h3.OrbitNodeKey, installUUID3)), http.StatusNoContent) + + s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/install/%d", h4.ID, titleID), nil, http.StatusAccepted, &resp) + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", h4.ID), nil, http.StatusOK, &getHostSoftwareResp) + require.Len(t, getHostSoftwareResp.Software, 1) + installUUID4a := getHostSoftwareResp.Software[0].LastInstall.InstallUUID + s.Do("POST", "/api/fleet/orbit/software_install/result", json.RawMessage(fmt.Sprintf(`{ + "orbit_node_key": %q, + "install_uuid": %q, + "pre_install_condition_output": "" + }`, *h4.OrbitNodeKey, installUUID4a)), http.StatusNoContent) + + s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/install/%d", h4.ID, titleID), nil, http.StatusAccepted, &resp) + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", h4.ID), nil, http.StatusOK, &getHostSoftwareResp) + require.Len(t, getHostSoftwareResp.Software, 1) + installUUID4b := getHostSoftwareResp.Software[0].LastInstall.InstallUUID + _ = installUUID4b + // status is reflected in software title response titleResp := getSoftwareTitleResponse{} s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/software/titles/%d", titleID), nil, http.StatusOK, &titleResp, "team_id", strconv.Itoa(int(*teamID))) @@ -9344,77 +9413,59 @@ func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerHostRequests() { require.Equal(t, "ruby.deb", titleResp.SoftwareTitle.SoftwarePackage.Name) require.NotNil(t, titleResp.SoftwareTitle.SoftwarePackage.Status) require.Equal(t, fleet.SoftwareInstallerStatusSummary{ - Installed: 0, - Pending: 1, - Failed: 0, + Installed: 1, + Pending: 2, + Failed: 1, }, *titleResp.SoftwareTitle.SoftwarePackage.Status) // status is reflected in list hosts responses and counts when filtering by software title and status - var listResp listHostsResponse - s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) - require.Len(t, listResp.Hosts, 1) - require.Equal(t, h.ID, listResp.Hosts[0].ID) - - var countResp countHostsResponse - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) - require.Equal(t, 1, countResp.Count) - - listResp = listHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "failed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) - require.Len(t, listResp.Hosts, 0) - - countResp = countHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "failed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) - require.Equal(t, 0, countResp.Count) - - listResp = listHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "installed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) - require.Len(t, listResp.Hosts, 0) - - countResp = countHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "installed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) - require.Equal(t, 0, countResp.Count) - + // create a label to test also the counts per label with the software install status filter var labelResp createLabelResponse s.DoJSON("POST", "/api/latest/fleet/labels", &createLabelRequest{fleet.LabelPayload{ Name: "test", - Hosts: []string{h.Hostname}, + Hosts: []string{h.Hostname, h2.Hostname, h3.Hostname, h4.Hostname}, }}, http.StatusOK, &labelResp) require.NotZero(t, labelResp.Label.ID) - listResp = listHostsResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp) - require.Len(t, listResp.Hosts, 1) - require.Equal(t, h.ID, listResp.Hosts[0].ID) + cases := []struct { + status string + count int + hostIDs []uint + }{ + {"pending", 2, []uint{h.ID, h4.ID}}, + {"failed", 1, []uint{h3.ID}}, + {"installed", 1, []uint{h2.ID}}, + } + for _, c := range cases { + t.Run(c.status, func(t *testing.T) { + var listResp listHostsResponse + s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", c.status, "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) + require.Len(t, listResp.Hosts, c.count) + gotIDs := make([]uint, 0, c.count) + for _, h := range listResp.Hosts { + gotIDs = append(gotIDs, h.ID) + } + require.ElementsMatch(t, c.hostIDs, gotIDs) - countResp = countHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) - require.Equal(t, 1, countResp.Count) + var countResp countHostsResponse + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", c.status, "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) + require.Equal(t, c.count, countResp.Count) - listResp = listHostsResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) - require.Len(t, listResp.Hosts, 1) - require.Equal(t, h.ID, listResp.Hosts[0].ID) + // count with label filter + countResp = countHostsResponse{} + s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", c.status, "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) + require.Equal(t, c.count, countResp.Count) - countResp = countHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "pending", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) - require.Equal(t, 1, countResp.Count) - - listResp = listHostsResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", "installed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) - require.Len(t, listResp.Hosts, 0) - - countResp = countHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "installed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) - require.Equal(t, 0, countResp.Count) - - listResp = listHostsResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", "failed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) - require.Len(t, listResp.Hosts, 0) - - countResp = countHostsResponse{} - s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "failed", "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID)), "label_id", strconv.Itoa(int(labelResp.Label.ID))) - require.Equal(t, 0, countResp.Count) + listResp = listHostsResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/labels/%d/hosts", labelResp.Label.ID), nil, http.StatusOK, &listResp, "software_status", c.status, "team_id", strconv.Itoa(int(*teamID)), "software_title_id", strconv.Itoa(int(titleID))) + require.Len(t, listResp.Hosts, c.count) + gotIDs = make([]uint, 0, c.count) + for _, h := range listResp.Hosts { + gotIDs = append(gotIDs, h.ID) + } + require.ElementsMatch(t, c.hostIDs, gotIDs) + }) + } // filter validations r := s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "uninstalled") @@ -9429,11 +9480,6 @@ func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerHostRequests() { require.Contains(t, extractServerErrorText(r.Body), "Invalid parameters. The combination of software_version_id and software_title_id is not allowed.") r = s.Do("GET", "/api/latest/fleet/hosts", nil, http.StatusBadRequest, "software_status", "installed", "team_id", "1", "software_title_id", "1", "software_id", "1") require.Contains(t, extractServerErrorText(r.Body), "Invalid parameters. The combination of software_id and software_title_id is not allowed.") - - // TODO(roberto): once we have endpoints to retrieve installers, - // request them using the orbit node key - - // TODO(sarah): test other statuses once we have endpoints to set results via orbit } func (s *integrationEnterpriseTestSuite) TestHostSoftwareInstallResult() { diff --git a/server/service/software_installers_test.go b/server/service/software_installers_test.go index 65329a332a..d58b41d963 100644 --- a/server/service/software_installers_test.go +++ b/server/service/software_installers_test.go @@ -60,7 +60,7 @@ func TestSoftwareInstallersAuth(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := viewer.NewContext(ctx, viewer.Viewer{User: tt.user}) - ds.GetSoftwareInstallerMetadataByTeamAndTitleIDFunc = func(ctx context.Context, teamID *uint, titleID uint) (*fleet.SoftwareInstaller, error) { + ds.GetSoftwareInstallerMetadataByTeamAndTitleIDFunc = func(ctx context.Context, teamID *uint, titleID uint, withScripts bool) (*fleet.SoftwareInstaller, error) { return &fleet.SoftwareInstaller{TeamID: tt.teamID}, nil } diff --git a/server/service/software_titles.go b/server/service/software_titles.go index b39f8c1767..0aeab6d2eb 100644 --- a/server/service/software_titles.go +++ b/server/service/software_titles.go @@ -176,7 +176,7 @@ func (svc *Service) SoftwareTitleByID(ctx context.Context, id uint, teamID *uint } if license.IsPremium() { // add software installer data - meta, err := svc.ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, teamID, id) + meta, err := svc.ds.GetSoftwareInstallerMetadataByTeamAndTitleID(ctx, teamID, id, true) if err != nil && !fleet.IsNotFound(err) { return nil, ctxerr.Wrap(ctx, err, "get software installer metadata") } From da85d91551cb7a3c34f0f98f03a7efd17ecf099b Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Wed, 15 May 2024 09:55:27 -0300 Subject: [PATCH 45/56] add counts to host software endpoints (#18995) part of #18677 # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Added/updated tests - [x] Manual QA for all new/changed functionality --- frontend/__mocks__/hostMock.ts | 1 + .../TableContainer/TableContainer.tsx | 3 ++ .../HostSoftwareTable/HostSoftwareTable.tsx | 20 +++++++++++++ frontend/services/entities/device_user.ts | 1 + frontend/services/entities/hosts.ts | 1 + server/datastore/mysql/software.go | 13 ++++++++- server/datastore/mysql/software_test.go | 29 ++++++++++--------- server/service/devices.go | 3 +- server/service/hosts.go | 3 +- 9 files changed, 57 insertions(+), 17 deletions(-) diff --git a/frontend/__mocks__/hostMock.ts b/frontend/__mocks__/hostMock.ts index 6800356227..fce7dec502 100644 --- a/frontend/__mocks__/hostMock.ts +++ b/frontend/__mocks__/hostMock.ts @@ -149,6 +149,7 @@ export const createMockHostSoftware = ( }; const DEFAULT_GET_HOST_SOFTWARE_RESPONSE_MOCK: IGetHostSoftwareResponse = { + count: 1, software: [createMockHostSoftware()], meta: { has_next_results: false, diff --git a/frontend/components/TableContainer/TableContainer.tsx b/frontend/components/TableContainer/TableContainer.tsx index 68fddb04cf..1ddcd7563a 100644 --- a/frontend/components/TableContainer/TableContainer.tsx +++ b/frontend/components/TableContainer/TableContainer.tsx @@ -68,6 +68,9 @@ interface ITableContainerProps { primarySelectAction?: IActionButtonProps; /** Secondary button/s after selecting a row */ secondarySelectActions?: IActionButtonProps[]; // TODO: Combine with primarySelectAction as these are all rendered in the same spot + /** + * @deprecated please use renderCount instead + * */ filteredCount?: number; searchToolTipText?: string; searchQueryColumn?: string; diff --git a/frontend/pages/hosts/details/cards/Software/HostSoftwareTable/HostSoftwareTable.tsx b/frontend/pages/hosts/details/cards/Software/HostSoftwareTable/HostSoftwareTable.tsx index 885c5947c6..f8e7a3c4f7 100644 --- a/frontend/pages/hosts/details/cards/Software/HostSoftwareTable/HostSoftwareTable.tsx +++ b/frontend/pages/hosts/details/cards/Software/HostSoftwareTable/HostSoftwareTable.tsx @@ -96,9 +96,29 @@ const HostSoftwareTable = ({ [determineQueryParamChange, pagePath, generateNewQueryParams, router] ); + const getItemsCountText = () => { + const count = data?.count; + if (!data?.software?.length || !count) return ""; + + return count === 1 ? `${count} software item` : `${count} software items`; + }; + + const renderSoftwareCount = () => { + const itemText = getItemsCountText(); + + if (!itemText) return null; + + return ( +
+ {itemText} +
+ ); + }; + return (
0} + metaData = &fleet.PaginationMetadata{ + HasPreviousResults: opts.Page > 0, + TotalResults: titleCount, + } if len(hostSoftwareList) > int(perPage) { metaData.HasNextResults = true hostSoftwareList = hostSoftwareList[:len(hostSoftwareList)-1] diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index 5494504dbb..2cc606601e 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -3040,7 +3040,7 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { // it now returns the software with vulnerabilities and installed paths sw, meta, err = ds.ListHostSoftware(ctx, host.ID, false, opts) require.NoError(t, err) - require.Equal(t, &fleet.PaginationMetadata{}, meta) + require.Equal(t, &fleet.PaginationMetadata{TotalResults: 5}, meta) compareResults([]*fleet.HostSoftwareWithInstaller{ expected["a1"], expected["a2"], expected["b"], expected["c"], expected["d"], }, sw) @@ -3200,7 +3200,7 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { // request without available software sw, meta, err = ds.ListHostSoftware(ctx, host.ID, false, opts) require.NoError(t, err) - require.Equal(t, &fleet.PaginationMetadata{}, meta) + require.Equal(t, &fleet.PaginationMetadata{TotalResults: 7}, meta) compareResults([]*fleet.HostSoftwareWithInstaller{ expected["a1"], expected["a2"], expected["b"], expected["c"], expected["d"], expected["i0"], expected["i1"], }, sw) @@ -3208,7 +3208,7 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { // request with available software sw, meta, err = ds.ListHostSoftware(ctx, host.ID, true, opts) require.NoError(t, err) - require.Equal(t, &fleet.PaginationMetadata{}, meta) + require.Equal(t, &fleet.PaginationMetadata{TotalResults: 8}, meta) compareResults([]*fleet.HostSoftwareWithInstaller{ expected["a1"], expected["a2"], expected["b"], expected["c"], expected["d"], expected["i0"], expected["i1"], expected["i2"], }, sw) @@ -3218,7 +3218,7 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { opts.TestSecondaryOrderDirection = fleet.OrderDescending sw, meta, err = ds.ListHostSoftware(ctx, host.ID, false, opts) require.NoError(t, err) - require.Equal(t, &fleet.PaginationMetadata{}, meta) + require.Equal(t, &fleet.PaginationMetadata{TotalResults: 7}, meta) compareResults([]*fleet.HostSoftwareWithInstaller{ expected["i1"], expected["i0"], expected["d"], expected["c"], expected["b"], expected["a2"], expected["a1"], }, sw) @@ -3267,7 +3267,7 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { // request without available software sw, meta, err = ds.ListHostSoftware(ctx, host.ID, false, opts) require.NoError(t, err) - require.Equal(t, &fleet.PaginationMetadata{}, meta) + require.Equal(t, &fleet.PaginationMetadata{TotalResults: 7}, meta) compareResults([]*fleet.HostSoftwareWithInstaller{ expected["a1"], expected["a2"], expected["b"], expected["c"], expected["d"], expected["i0"], expected["i1"], }, sw) @@ -3275,7 +3275,7 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { // request with available software sw, meta, err = ds.ListHostSoftware(ctx, host.ID, true, opts) require.NoError(t, err) - require.Equal(t, &fleet.PaginationMetadata{}, meta) + require.Equal(t, &fleet.PaginationMetadata{TotalResults: 8}, meta) compareResults([]*fleet.HostSoftwareWithInstaller{ expected["a1"], expected["a2"], expected["b"], expected["c"], expected["d"], expected["i0"], expected["i1"], expected["i2"], }, sw) @@ -3292,8 +3292,9 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { require.Equal(t, &fleet.PaginationMetadata{}, meta) // sees the available installer in its team - sw, _, err = ds.ListHostSoftware(ctx, tmHost.ID, true, opts) + sw, meta, err = ds.ListHostSoftware(ctx, tmHost.ID, true, opts) require.NoError(t, err) + require.Equal(t, &fleet.PaginationMetadata{TotalResults: 1}, meta) compareResults([]*fleet.HostSoftwareWithInstaller{expected["i3"]}, sw) // test with a search query (searches on name), with and without available software @@ -3324,43 +3325,43 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { opts: fleet.ListOptions{PerPage: 3}, withAvailable: false, wantNames: []string{expected["a1"].Name, expected["a2"].Name, expected["b"].Name}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false, TotalResults: 7}, }, { opts: fleet.ListOptions{Page: 1, PerPage: 3}, withAvailable: false, wantNames: []string{expected["c"].Name, expected["d"].Name, expected["i0"].Name}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: true}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: true, TotalResults: 7}, }, { opts: fleet.ListOptions{Page: 2, PerPage: 3}, withAvailable: false, wantNames: []string{expected["i1"].Name}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 7}, }, { opts: fleet.ListOptions{Page: 3, PerPage: 3}, withAvailable: false, wantNames: []string{}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 7}, }, { opts: fleet.ListOptions{PerPage: 4}, withAvailable: true, wantNames: []string{expected["a1"].Name, expected["a2"].Name, expected["b"].Name, expected["c"].Name}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false, TotalResults: 8}, }, { opts: fleet.ListOptions{Page: 1, PerPage: 4}, withAvailable: true, wantNames: []string{expected["d"].Name, expected["i0"].Name, expected["i1"].Name, expected["i2"].Name}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 8}, }, { opts: fleet.ListOptions{Page: 2, PerPage: 4}, withAvailable: true, wantNames: []string{}, - wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true}, + wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 8}, }, } for _, c := range cases { diff --git a/server/service/devices.go b/server/service/devices.go index d0a319b8b1..41cd5dc8dd 100644 --- a/server/service/devices.go +++ b/server/service/devices.go @@ -606,6 +606,7 @@ func (r *getDeviceSoftwareRequest) deviceAuthToken() string { type getDeviceSoftwareResponse struct { Software []*fleet.HostSoftwareWithInstaller `json:"software"` + Count int `json:"count"` Meta *fleet.PaginationMetadata `json:"meta,omitempty"` Err error `json:"error,omitempty"` } @@ -627,5 +628,5 @@ func getDeviceSoftwareEndpoint(ctx context.Context, request interface{}, svc fle if res == nil { res = []*fleet.HostSoftwareWithInstaller{} } - return getDeviceSoftwareResponse{Software: res, Meta: meta}, nil + return getDeviceSoftwareResponse{Software: res, Meta: meta, Count: int(meta.TotalResults)}, nil } diff --git a/server/service/hosts.go b/server/service/hosts.go index e81f211d18..90a3e20994 100644 --- a/server/service/hosts.go +++ b/server/service/hosts.go @@ -2479,6 +2479,7 @@ type getHostSoftwareRequest struct { type getHostSoftwareResponse struct { Software []*fleet.HostSoftwareWithInstaller `json:"software"` + Count int `json:"count"` Meta *fleet.PaginationMetadata `json:"meta,omitempty"` Err error `json:"error,omitempty"` } @@ -2494,7 +2495,7 @@ func getHostSoftwareEndpoint(ctx context.Context, request interface{}, svc fleet if res == nil { res = []*fleet.HostSoftwareWithInstaller{} } - return getHostSoftwareResponse{Software: res, Meta: meta}, nil + return getHostSoftwareResponse{Software: res, Meta: meta, Count: int(meta.TotalResults)}, nil } func (svc *Service) ListHostSoftware(ctx context.Context, hostID uint, opts fleet.ListOptions) ([]*fleet.HostSoftwareWithInstaller, *fleet.PaginationMetadata, error) { From 6e8e1bd0b49f212455ac637531d7f6b6148ed049 Mon Sep 17 00:00:00 2001 From: Dante Catalfamo <43040593+dantecatalfamo@users.noreply.github.com> Date: Wed, 15 May 2024 10:31:13 -0400 Subject: [PATCH 46/56] Install script extension (#19012) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Because windows won't run powershell scripts without it 👎 --- orbit/pkg/installer/installer.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/orbit/pkg/installer/installer.go b/orbit/pkg/installer/installer.go index 46cd53fb2e..4109dcea61 100644 --- a/orbit/pkg/installer/installer.go +++ b/orbit/pkg/installer/installer.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "sync" "time" @@ -212,7 +213,11 @@ func (r *Runner) installSoftware(ctx context.Context, installID string) (*fleet. } }() - installOutput, installExitCode, err := r.runInstallerScript(ctx, installer.InstallScript, installerPath, "install-script") + scriptExtension := ".sh" + if runtime.GOOS == "windows" { + scriptExtension = ".ps1" + } + installOutput, installExitCode, err := r.runInstallerScript(ctx, installer.InstallScript, installerPath, "install-script"+scriptExtension) payload.InstallScriptOutput = &installOutput payload.InstallScriptExitCode = &installExitCode if err != nil { @@ -220,7 +225,7 @@ func (r *Runner) installSoftware(ctx context.Context, installID string) (*fleet. } if installer.PostInstallScript != "" { - postOutput, postExitCode, postErr := r.runInstallerScript(ctx, installer.PostInstallScript, installerPath, "post-install-script") + postOutput, postExitCode, postErr := r.runInstallerScript(ctx, installer.PostInstallScript, installerPath, "post-install-script"+scriptExtension) payload.PostInstallScriptOutput = &postOutput payload.PostInstallScriptExitCode = &postExitCode From 5fb52f6baf20abb718347501bdd08d3ea5b1e5f5 Mon Sep 17 00:00:00 2001 From: Dante Catalfamo <43040593+dantecatalfamo@users.noreply.github.com> Date: Wed, 15 May 2024 12:17:18 -0400 Subject: [PATCH 47/56] Install script extension test (#19025) Fixes a test I accidentally broke in #19012 --- orbit/pkg/installer/installer_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/orbit/pkg/installer/installer_test.go b/orbit/pkg/installer/installer_test.go index 7b61290269..7b75a7adbb 100644 --- a/orbit/pkg/installer/installer_test.go +++ b/orbit/pkg/installer/installer_test.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strconv" "testing" @@ -285,8 +286,12 @@ func TestInstallerRun(t *testing.T) { require.True(t, tmpDirFnCalled) require.True(t, execCalled) - require.Contains(t, executedScripts, filepath.Join(tmpDir, "install-script")) - require.Contains(t, executedScripts, filepath.Join(tmpDir, "post-install-script")) + scriptExtension := ".sh" + if runtime.GOOS == "windows" { + scriptExtension = ".ps1" + } + require.Contains(t, executedScripts, filepath.Join(tmpDir, "install-script"+scriptExtension)) + require.Contains(t, executedScripts, filepath.Join(tmpDir, "post-install-script"+scriptExtension)) require.Contains(t, execEnv, "INSTALLER_PATH="+filepath.Join(tmpDir, strconv.Itoa(int(installDetails.InstallerID))+".pkg")) require.True(t, queryFnCalled) From 40dc8e57ed45a54022cb73b89b47a13dbb3da9c0 Mon Sep 17 00:00:00 2001 From: Jahziel Villasana-Espinoza Date: Wed, 15 May 2024 13:41:35 -0400 Subject: [PATCH 48/56] fix: add missing software_package field (#18998) No related issue, just cleanup work on the feature # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [x] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements) - [x] Added/updated tests - [x] Manual QA for all new/changed functionality --- changes/jve-fix-software-package | 1 + cmd/fleetctl/get_test.go | 4 +- server/datastore/mysql/software_titles.go | 10 +-- .../datastore/mysql/software_titles_test.go | 45 ++++++------ server/fleet/datastore.go | 2 +- server/fleet/service.go | 2 +- server/fleet/software.go | 24 +++++++ server/mock/datastore_mock.go | 4 +- server/service/client_software.go | 2 +- server/service/integration_enterprise_test.go | 71 +++++++++++++++---- server/service/software_titles.go | 14 ++-- server/service/software_titles_test.go | 4 +- 12 files changed, 128 insertions(+), 55 deletions(-) create mode 100644 changes/jve-fix-software-package diff --git a/changes/jve-fix-software-package b/changes/jve-fix-software-package new file mode 100644 index 0000000000..ca614bff40 --- /dev/null +++ b/changes/jve-fix-software-package @@ -0,0 +1 @@ +- Adds a missing field `software_package` to the response from the List Software Titles endpoint. \ No newline at end of file diff --git a/cmd/fleetctl/get_test.go b/cmd/fleetctl/get_test.go index e961ce8e48..5fbc221fb7 100644 --- a/cmd/fleetctl/get_test.go +++ b/cmd/fleetctl/get_test.go @@ -623,9 +623,9 @@ func TestGetSoftwareTitles(t *testing.T) { var gotTeamID *uint - ds.ListSoftwareTitlesFunc = func(ctx context.Context, opt fleet.SoftwareTitleListOptions, tmFilter fleet.TeamFilter) ([]fleet.SoftwareTitle, int, *fleet.PaginationMetadata, error) { + ds.ListSoftwareTitlesFunc = func(ctx context.Context, opt fleet.SoftwareTitleListOptions, tmFilter fleet.TeamFilter) ([]fleet.SoftwareTitleListResult, int, *fleet.PaginationMetadata, error) { gotTeamID = opt.TeamID - return []fleet.SoftwareTitle{ + return []fleet.SoftwareTitleListResult{ { Name: "foo", Source: "chrome_extensions", diff --git a/server/datastore/mysql/software_titles.go b/server/datastore/mysql/software_titles.go index edf3066b67..bd98f58b03 100644 --- a/server/datastore/mysql/software_titles.go +++ b/server/datastore/mysql/software_titles.go @@ -69,7 +69,7 @@ func (ds *Datastore) ListSoftwareTitles( ctx context.Context, opt fleet.SoftwareTitleListOptions, tmFilter fleet.TeamFilter, -) ([]fleet.SoftwareTitle, int, *fleet.PaginationMetadata, error) { +) ([]fleet.SoftwareTitleListResult, int, *fleet.PaginationMetadata, error) { if opt.ListOptions.After != "" { return nil, 0, nil, fleet.NewInvalidArgumentError("after", "not supported for software titles") } @@ -93,7 +93,7 @@ func (ds *Datastore) ListSoftwareTitles( getTitlesCountStmt := fmt.Sprintf(`SELECT COUNT(DISTINCT s.id) FROM (%s) AS s`, getTitlesStmt) // grab titles that match the list options - var titles []fleet.SoftwareTitle + var titles []fleet.SoftwareTitleListResult getTitlesStmt, args = appendListOptionsWithCursorToSQL(getTitlesStmt, args, &opt.ListOptions) // appendListOptionsWithCursorToSQL doesn't support multicolumn sort, so // we need to add it here @@ -201,8 +201,10 @@ SELECT st.source, st.browser, MAX(COALESCE(sthc.hosts_count, 0)) as hosts_count, - MAX(COALESCE(sthc.updated_at, date('0001-01-01 00:00:00'))) as counts_updated_at + MAX(COALESCE(sthc.updated_at, date('0001-01-01 00:00:00'))) as counts_updated_at, + si.filename as software_package FROM software_titles st +LEFT JOIN software_installers si ON si.title_id = st.id LEFT JOIN software_titles_host_counts sthc ON sthc.software_title_id = st.id AND sthc.team_id = ? -- placeholder for JOIN on software/software_cve %s @@ -210,7 +212,7 @@ LEFT JOIN software_titles_host_counts sthc ON sthc.software_title_id = st.id AND WHERE %s -- placeholder for filter based on software installed on hosts + software installers AND (%s) -GROUP BY st.id` +GROUP BY st.id, software_package` cveJoinType := "LEFT" if opt.VulnerableOnly { diff --git a/server/datastore/mysql/software_titles_test.go b/server/datastore/mysql/software_titles_test.go index a6eb1a7b67..ea70177055 100644 --- a/server/datastore/mysql/software_titles_test.go +++ b/server/datastore/mysql/software_titles_test.go @@ -39,10 +39,10 @@ func TestSoftwareTitles(t *testing.T) { func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { ctx := context.Background() - cmpNameVersionCount := func(want, got []fleet.SoftwareTitle) { - cmp := make([]fleet.SoftwareTitle, len(got)) + cmpNameVersionCount := func(want, got []fleet.SoftwareTitleListResult) { + cmp := make([]fleet.SoftwareTitleListResult, len(got)) for i, sw := range got { - cmp[i] = fleet.SoftwareTitle{Name: sw.Name, HostsCount: sw.HostsCount} + cmp[i] = fleet.SoftwareTitleListResult{Name: sw.Name, HostsCount: sw.HostsCount} } require.ElementsMatch(t, want, cmp) } @@ -80,7 +80,7 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { globalOpts := fleet.SoftwareTitleListOptions{ListOptions: fleet.ListOptions{OrderKey: "hosts_count", OrderDirection: fleet.OrderDescending}} globalCounts := listSoftwareTitlesCheckCount(t, ds, 2, 2, globalOpts) - want := []fleet.SoftwareTitle{ + want := []fleet.SoftwareTitleListResult{ {Name: "foo", HostsCount: 2}, {Name: "bar", HostsCount: 1}, } @@ -99,7 +99,7 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) globalCounts = listSoftwareTitlesCheckCount(t, ds, 1, 1, globalOpts) - want = []fleet.SoftwareTitle{ + want = []fleet.SoftwareTitleListResult{ {Name: "foo", HostsCount: 2}, } cmpNameVersionCount(want, globalCounts) @@ -111,7 +111,7 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { // listing does not return the new software title entry allSw := listSoftwareTitlesCheckCount(t, ds, 1, 1, fleet.SoftwareTitleListOptions{}) - want = []fleet.SoftwareTitle{ + want = []fleet.SoftwareTitleListResult{ {Name: "foo", HostsCount: 2}, } cmpNameVersionCount(want, allSw) @@ -144,7 +144,7 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { // at this point, there's no counts per team, only global counts globalCounts = listSoftwareTitlesCheckCount(t, ds, 1, 1, globalOpts) - want = []fleet.SoftwareTitle{ + want = []fleet.SoftwareTitleListResult{ {Name: "foo", HostsCount: 2}, } cmpNameVersionCount(want, globalCounts) @@ -155,7 +155,7 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { ListOptions: fleet.ListOptions{OrderKey: "hosts_count", OrderDirection: fleet.OrderDescending}, } team1Counts := listSoftwareTitlesCheckCount(t, ds, 0, 0, team1Opts) - want = []fleet.SoftwareTitle{} + want = []fleet.SoftwareTitleListResult{} cmpNameVersionCount(want, team1Counts) checkTableTotalCount(1) @@ -165,14 +165,14 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) globalCounts = listSoftwareTitlesCheckCount(t, ds, 2, 2, globalOpts) - want = []fleet.SoftwareTitle{ + want = []fleet.SoftwareTitleListResult{ {Name: "foo", HostsCount: 4}, {Name: "bar", HostsCount: 1}, } cmpNameVersionCount(want, globalCounts) team1Counts = listSoftwareTitlesCheckCount(t, ds, 1, 1, team1Opts) - want = []fleet.SoftwareTitle{ + want = []fleet.SoftwareTitleListResult{ {Name: "foo", HostsCount: 2}, } cmpNameVersionCount(want, team1Counts) @@ -185,7 +185,7 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { ListOptions: fleet.ListOptions{OrderKey: "hosts_count", OrderDirection: fleet.OrderDescending}, } team2Counts := listSoftwareTitlesCheckCount(t, ds, 2, 2, team2Opts) - want = []fleet.SoftwareTitle{ + want = []fleet.SoftwareTitleListResult{ {Name: "foo", HostsCount: 1}, {Name: "bar", HostsCount: 1}, } @@ -203,19 +203,19 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) globalCounts = listSoftwareTitlesCheckCount(t, ds, 1, 1, globalOpts) - want = []fleet.SoftwareTitle{ + want = []fleet.SoftwareTitleListResult{ {Name: "foo", HostsCount: 4}, } cmpNameVersionCount(want, globalCounts) team1Counts = listSoftwareTitlesCheckCount(t, ds, 1, 1, team1Opts) - want = []fleet.SoftwareTitle{ + want = []fleet.SoftwareTitleListResult{ {Name: "foo", HostsCount: 2}, } cmpNameVersionCount(want, team1Counts) team2Counts = listSoftwareTitlesCheckCount(t, ds, 1, 1, team2Opts) - want = []fleet.SoftwareTitle{ + want = []fleet.SoftwareTitleListResult{ {Name: "foo", HostsCount: 1}, } cmpNameVersionCount(want, team2Counts) @@ -240,13 +240,13 @@ func testSoftwareSyncHostsSoftwareTitles(t *testing.T, ds *Datastore) { require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) globalCounts = listSoftwareTitlesCheckCount(t, ds, 1, 1, globalOpts) - want = []fleet.SoftwareTitle{ + want = []fleet.SoftwareTitleListResult{ {Name: "foo", HostsCount: 3}, } cmpNameVersionCount(want, globalCounts) team1Counts = listSoftwareTitlesCheckCount(t, ds, 1, 1, team1Opts) - want = []fleet.SoftwareTitle{ + want = []fleet.SoftwareTitleListResult{ {Name: "foo", HostsCount: 2}, } cmpNameVersionCount(want, team1Counts) @@ -457,7 +457,7 @@ func testOrderSoftwareTitles(t *testing.T, ds *Datastore) { require.Equal(t, "apps", titles[1].Source) } -func listSoftwareTitlesCheckCount(t *testing.T, ds *Datastore, expectedListCount int, expectedFullCount int, opts fleet.SoftwareTitleListOptions) []fleet.SoftwareTitle { +func listSoftwareTitlesCheckCount(t *testing.T, ds *Datastore, expectedListCount int, expectedFullCount int, opts fleet.SoftwareTitleListOptions) []fleet.SoftwareTitleListResult { titles, count, _, err := ds.ListSoftwareTitles(context.Background(), opts, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}}) require.NoError(t, err) require.Len(t, titles, expectedListCount) @@ -549,7 +549,7 @@ func testTeamFilterSoftwareTitles(t *testing.T, ds *Datastore) { require.NoError(t, err) // ListSoftwareTitles does not populate version host counts, so we do that manually titles[0].Versions[0].HostsCount = ptr.Uint(1) - assert.Equal(t, titles[0], *title) + assert.Equal(t, titles[0], fleet.SoftwareTitleListResult{ID: title.ID, Name: title.Name, Source: title.Source, Browser: title.Browser, HostsCount: title.HostsCount, VersionsCount: title.VersionsCount, Versions: title.Versions, CountsUpdatedAt: title.CountsUpdatedAt}) // Testing with team filter -- this team does not contain this software title _, err = ds.SoftwareTitleByID(context.Background(), titles[0].ID, &team1.ID, globalTeamFilter) @@ -585,7 +585,7 @@ func testTeamFilterSoftwareTitles(t *testing.T, ds *Datastore) { require.NoError(t, err) // ListSoftwareTitles does not populate version host counts, so we do that manually titles[0].Versions[0].HostsCount = ptr.Uint(1) - assert.Equal(t, titles[0], *title) + assert.Equal(t, titles[0], fleet.SoftwareTitleListResult{ID: title.ID, Name: title.Name, Source: title.Source, Browser: title.Browser, HostsCount: title.HostsCount, VersionsCount: title.VersionsCount, Versions: title.Versions, CountsUpdatedAt: title.CountsUpdatedAt}) // Testing the team 2 user titles, count, _, err = ds.ListSoftwareTitles(context.Background(), fleet.SoftwareTitleListOptions{ListOptions: fleet.ListOptions{}, TeamID: &team2.ID}, fleet.TeamFilter{ @@ -607,7 +607,7 @@ func testTeamFilterSoftwareTitles(t *testing.T, ds *Datastore) { require.Equal(t, uint(0), titles[2].VersionsCount) } -func sortTitlesByName(titles []fleet.SoftwareTitle) { +func sortTitlesByName(titles []fleet.SoftwareTitleListResult) { sort.Slice(titles, func(i, j int) bool { return titles[i].Name < titles[j].Name }) } @@ -645,6 +645,10 @@ func testListSoftwareTitlesInstallersOnly(t *testing.T, ds *Datastore) { require.Equal(t, "apps", titles[1].Source) require.True(t, titles[0].CountsUpdatedAt.IsZero()) require.True(t, titles[1].CountsUpdatedAt.IsZero()) + require.NotNil(t, titles[0].SoftwarePackage) + require.Equal(t, "installer1.pkg", *titles[0].SoftwarePackage) + require.NotNil(t, titles[1].SoftwarePackage) + require.Equal(t, "installer2.pkg", *titles[1].SoftwarePackage) require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now())) require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) @@ -688,7 +692,6 @@ func testListSoftwareTitlesInstallersOnly(t *testing.T, ds *Datastore) { require.EqualValues(t, 2, counts) require.Len(t, titles, 2) require.True(t, titles[0].CountsUpdatedAt.IsZero()) - } func testListSoftwareTitlesAvailableForInstallFilter(t *testing.T, ds *Datastore) { diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 76aaa3a3fa..a17bcfcf85 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -487,7 +487,7 @@ type Datastore interface { /////////////////////////////////////////////////////////////////////////////// // Software Titles - ListSoftwareTitles(ctx context.Context, opt SoftwareTitleListOptions, tmFilter TeamFilter) ([]SoftwareTitle, int, *PaginationMetadata, error) + ListSoftwareTitles(ctx context.Context, opt SoftwareTitleListOptions, tmFilter TeamFilter) ([]SoftwareTitleListResult, int, *PaginationMetadata, error) SoftwareTitleByID(ctx context.Context, id uint, teamID *uint, tmFilter TeamFilter) (*SoftwareTitle, error) // InsertSoftwareInstallRequest tracks a new request to install the provided diff --git a/server/fleet/service.go b/server/fleet/service.go index f657d96a8c..f90d8b32c3 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -629,7 +629,7 @@ type Service interface { // ///////////////////////////////////////////////////////////////////////////// // Software Titles - ListSoftwareTitles(ctx context.Context, opt SoftwareTitleListOptions) ([]SoftwareTitle, int, *PaginationMetadata, error) + ListSoftwareTitles(ctx context.Context, opt SoftwareTitleListOptions) ([]SoftwareTitleListResult, int, *PaginationMetadata, error) SoftwareTitleByID(ctx context.Context, id uint, teamID *uint) (*SoftwareTitle, error) // InstallSoftwareTitle installs a software title in the given host. diff --git a/server/fleet/software.go b/server/fleet/software.go index 4ab963b75b..0842d74933 100644 --- a/server/fleet/software.go +++ b/server/fleet/software.go @@ -162,6 +162,30 @@ type SoftwareTitle struct { SoftwarePackage *SoftwareInstaller `json:"software_package" db:"-"` } +// This type is essentially the same as the above SoftwareTitle type. The only difference is that +// SoftwarePackage is a string pointer here. This type is for use when listing out SoftwareTitles; +// the above type is used when fetching them individually. +type SoftwareTitleListResult struct { + ID uint `json:"id" db:"id"` + // Name is the name reported by osquery. + Name string `json:"name" db:"name"` + // Source is the source reported by osquery. + Source string `json:"source" db:"source"` + // Browser is the browser type (e.g., "chrome", "firefox", "safari") + Browser string `json:"browser,omitempty" db:"browser"` + // HostsCount is the number of hosts that use this software title. + HostsCount uint `json:"hosts_count" db:"hosts_count"` + // VesionsCount is the number of versions that have the same title. + VersionsCount uint `json:"versions_count" db:"versions_count"` + // Versions countains information about the versions that use this title. + Versions []SoftwareVersion `json:"versions" db:"-"` + // CountsUpdatedAt is the timestamp when the hosts count + // was last updated for that software title + CountsUpdatedAt *time.Time `json:"-" db:"counts_updated_at"` + // SoftwarePackage is the filename of the installer for this software title. + SoftwarePackage *string `json:"software_package" db:"software_package"` +} + type SoftwareTitleListOptions struct { // ListOptions cannot be embedded in order to unmarshall with validation. ListOptions ListOptions `url:"list_options"` diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index be3e08a7c3..b7a3c8ff6a 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -361,7 +361,7 @@ type DeleteIntegrationsFromTeamsFunc func(ctx context.Context, deletedIntgs flee type TeamExistsFunc func(ctx context.Context, teamID uint) (bool, error) -type ListSoftwareTitlesFunc func(ctx context.Context, opt fleet.SoftwareTitleListOptions, tmFilter fleet.TeamFilter) ([]fleet.SoftwareTitle, int, *fleet.PaginationMetadata, error) +type ListSoftwareTitlesFunc func(ctx context.Context, opt fleet.SoftwareTitleListOptions, tmFilter fleet.TeamFilter) ([]fleet.SoftwareTitleListResult, int, *fleet.PaginationMetadata, error) type SoftwareTitleByIDFunc func(ctx context.Context, id uint, teamID *uint, tmFilter fleet.TeamFilter) (*fleet.SoftwareTitle, error) @@ -3550,7 +3550,7 @@ func (s *DataStore) TeamExists(ctx context.Context, teamID uint) (bool, error) { return s.TeamExistsFunc(ctx, teamID) } -func (s *DataStore) ListSoftwareTitles(ctx context.Context, opt fleet.SoftwareTitleListOptions, tmFilter fleet.TeamFilter) ([]fleet.SoftwareTitle, int, *fleet.PaginationMetadata, error) { +func (s *DataStore) ListSoftwareTitles(ctx context.Context, opt fleet.SoftwareTitleListOptions, tmFilter fleet.TeamFilter) ([]fleet.SoftwareTitleListResult, int, *fleet.PaginationMetadata, error) { s.mu.Lock() s.ListSoftwareTitlesFuncInvoked = true s.mu.Unlock() diff --git a/server/service/client_software.go b/server/service/client_software.go index 90e738aa91..22c602e96c 100644 --- a/server/service/client_software.go +++ b/server/service/client_software.go @@ -16,7 +16,7 @@ func (c *Client) ListSoftwareVersions(query string) ([]fleet.Software, error) { } // ListSoftwareTitles retrieves the software titles installed on hosts. -func (c *Client) ListSoftwareTitles(query string) ([]fleet.SoftwareTitle, error) { +func (c *Client) ListSoftwareTitles(query string) ([]fleet.SoftwareTitleListResult, error) { verb, path := "GET", "/api/latest/fleet/software/titles" var responseBody listSoftwareTitlesResponse err := c.authenticatedRequestWithQuery(nil, verb, path, &responseBody, query) diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index afa6454db2..32325d5a57 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -6984,6 +6984,30 @@ func (s *integrationEnterpriseTestSuite) TestAllSoftwareTitles() { ctx := context.Background() t := s.T() + softwareTitleListResultsMatch := func(want, got []fleet.SoftwareTitleListResult) { + // compare only the fields we care about + for i := range got { + require.NotZero(t, got[i].ID) + got[i].ID = 0 + + for j := range got[i].Versions { + require.NotZero(t, got[i].Versions[j].ID) + got[i].Versions[j].ID = 0 + } + } + + // sort and use EqualValues instead of ElementsMatch in order + // to do a deep comparison of nested structures + sort.Slice(got, func(i, j int) bool { + return got[i].Name < got[j].Name + }) + sort.Slice(want, func(i, j int) bool { + return want[i].Name < want[j].Name + }) + + require.EqualValues(t, want, got) + } + softwareTitlesMatch := func(want, got []fleet.SoftwareTitle) { // compare only the fields we care about for i := range got { @@ -7085,7 +7109,7 @@ func (s *integrationEnterpriseTestSuite) TestAllSoftwareTitles() { s.DoJSON("GET", "/api/latest/fleet/software/titles", listSoftwareTitlesRequest{}, http.StatusOK, &resp) require.Equal(t, 2, resp.Count) require.NotEmpty(t, resp.CountsUpdatedAt) - softwareTitlesMatch([]fleet.SoftwareTitle{ + softwareTitleListResultsMatch([]fleet.SoftwareTitleListResult{ { Name: "foo", Source: "homebrew", @@ -7120,7 +7144,7 @@ func (s *integrationEnterpriseTestSuite) TestAllSoftwareTitles() { ) require.Equal(t, 2, resp.Count) require.NotEmpty(t, resp.CountsUpdatedAt) - softwareTitlesMatch([]fleet.SoftwareTitle{ + softwareTitleListResultsMatch([]fleet.SoftwareTitleListResult{ { Name: "foo", Source: "homebrew", @@ -7146,7 +7170,7 @@ func (s *integrationEnterpriseTestSuite) TestAllSoftwareTitles() { ) require.Equal(t, 2, resp.Count) require.NotEmpty(t, resp.CountsUpdatedAt) - softwareTitlesMatch([]fleet.SoftwareTitle{ + softwareTitleListResultsMatch([]fleet.SoftwareTitleListResult{ { Name: "bar", Source: "apps", @@ -7171,7 +7195,7 @@ func (s *integrationEnterpriseTestSuite) TestAllSoftwareTitles() { ) require.Equal(t, 2, resp.Count) require.Empty(t, resp.CountsUpdatedAt) - softwareTitlesMatch([]fleet.SoftwareTitle{}, resp.SoftwareTitles) + softwareTitleListResultsMatch([]fleet.SoftwareTitleListResult{}, resp.SoftwareTitles) // asking for vulnerable only software returns the expected values resp = listSoftwareTitlesResponse{} @@ -7183,7 +7207,7 @@ func (s *integrationEnterpriseTestSuite) TestAllSoftwareTitles() { ) require.Equal(t, 1, resp.Count) require.NotEmpty(t, resp.CountsUpdatedAt) - softwareTitlesMatch([]fleet.SoftwareTitle{ + softwareTitleListResultsMatch([]fleet.SoftwareTitleListResult{ { Name: "bar", Source: "apps", @@ -7205,7 +7229,7 @@ func (s *integrationEnterpriseTestSuite) TestAllSoftwareTitles() { ) require.Equal(t, 0, resp.Count) require.Empty(t, resp.CountsUpdatedAt) - softwareTitlesMatch([]fleet.SoftwareTitle{}, resp.SoftwareTitles) + softwareTitleListResultsMatch([]fleet.SoftwareTitleListResult{}, resp.SoftwareTitles) // add new software for tmHost software = []fleet.Software{ @@ -7233,7 +7257,7 @@ func (s *integrationEnterpriseTestSuite) TestAllSoftwareTitles() { ) require.Equal(t, 2, resp.Count) require.NotEmpty(t, resp.CountsUpdatedAt) - softwareTitlesMatch([]fleet.SoftwareTitle{ + softwareTitleListResultsMatch([]fleet.SoftwareTitleListResult{ { Name: "baz", Source: "deb_packages", @@ -7266,7 +7290,7 @@ func (s *integrationEnterpriseTestSuite) TestAllSoftwareTitles() { ) require.Equal(t, 3, resp.Count) require.NotEmpty(t, resp.CountsUpdatedAt) - softwareTitlesMatch([]fleet.SoftwareTitle{ + softwareTitleListResultsMatch([]fleet.SoftwareTitleListResult{ { Name: "baz", Source: "deb_packages", @@ -7306,7 +7330,7 @@ func (s *integrationEnterpriseTestSuite) TestAllSoftwareTitles() { "query", "123", ) require.Equal(t, 1, resp.Count) - softwareTitlesMatch([]fleet.SoftwareTitle{ + softwareTitleListResultsMatch([]fleet.SoftwareTitleListResult{ { Name: "bar", Source: "apps", @@ -7328,7 +7352,7 @@ func (s *integrationEnterpriseTestSuite) TestAllSoftwareTitles() { ) require.Equal(t, 2, resp.Count) require.NotEmpty(t, resp.CountsUpdatedAt) - softwareTitlesMatch([]fleet.SoftwareTitle{ + softwareTitleListResultsMatch([]fleet.SoftwareTitleListResult{ { Name: "bar", Source: "apps", @@ -7359,7 +7383,7 @@ func (s *integrationEnterpriseTestSuite) TestAllSoftwareTitles() { ) require.Equal(t, 2, resp.Count) require.NotEmpty(t, resp.CountsUpdatedAt) - softwareTitlesMatch([]fleet.SoftwareTitle{ + softwareTitleListResultsMatch([]fleet.SoftwareTitleListResult{ { Name: "bar", Source: "apps", @@ -7390,7 +7414,7 @@ func (s *integrationEnterpriseTestSuite) TestAllSoftwareTitles() { ) require.Equal(t, 2, resp.Count) require.NotEmpty(t, resp.CountsUpdatedAt) - softwareTitlesMatch([]fleet.SoftwareTitle{ + softwareTitleListResultsMatch([]fleet.SoftwareTitleListResult{ { Name: "bar", Source: "apps", @@ -7421,7 +7445,7 @@ func (s *integrationEnterpriseTestSuite) TestAllSoftwareTitles() { ) require.Equal(t, 2, resp.Count) require.NotEmpty(t, resp.CountsUpdatedAt) - softwareTitlesMatch([]fleet.SoftwareTitle{ + softwareTitleListResultsMatch([]fleet.SoftwareTitleListResult{ { Name: "bar", Source: "apps", @@ -7582,6 +7606,25 @@ func (s *integrationEnterpriseTestSuite) TestAllSoftwareTitles() { "GET", fmt.Sprintf("/api/latest/fleet/software/titles/%d", barTitle.ID), getSoftwareTitleRequest{}, http.StatusNotFound, &stResp, "team_id", "99999", ) + + // verify that software installers contain SoftwarePackage field + payload := &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "install", + Filename: "ruby.deb", + } + s.uploadSoftwareInstaller(payload, http.StatusOK, "") + + resp = listSoftwareTitlesResponse{} + s.DoJSON( + "GET", "/api/latest/fleet/software/titles", + listSoftwareTitlesRequest{}, + http.StatusOK, &resp, + "query", "ruby", + ) + + require.Len(t, resp.SoftwareTitles, 1) + require.NotNil(t, resp.SoftwareTitles[0].SoftwarePackage) + require.Equal(t, "ruby.deb", *resp.SoftwareTitles[0].SoftwarePackage) } func (s *integrationEnterpriseTestSuite) TestLockUnlockWipeWindowsLinux() { @@ -7865,7 +7908,7 @@ func (s *integrationEnterpriseTestSuite) TestSoftwareAuth() { var listSoftwareTitlesResp listSoftwareTitlesResponse s.DoJSON("GET", "/api/latest/fleet/software/titles", listSoftwareTitlesRequest{}, http.StatusOK, &listSoftwareTitlesResp) - var softwareFoo, softwareBar *fleet.SoftwareTitle + var softwareFoo, softwareBar *fleet.SoftwareTitleListResult for _, s := range listSoftwareTitlesResp.SoftwareTitles { s := s switch s.Name { diff --git a/server/service/software_titles.go b/server/service/software_titles.go index 0aeab6d2eb..a2776b22cd 100644 --- a/server/service/software_titles.go +++ b/server/service/software_titles.go @@ -21,11 +21,11 @@ type listSoftwareTitlesRequest struct { } type listSoftwareTitlesResponse struct { - Meta *fleet.PaginationMetadata `json:"meta"` - Count int `json:"count"` - CountsUpdatedAt *time.Time `json:"counts_updated_at"` - SoftwareTitles []fleet.SoftwareTitle `json:"software_titles"` - Err error `json:"error,omitempty"` + Meta *fleet.PaginationMetadata `json:"meta"` + Count int `json:"count"` + CountsUpdatedAt *time.Time `json:"counts_updated_at"` + SoftwareTitles []fleet.SoftwareTitleListResult `json:"software_titles"` + Err error `json:"error,omitempty"` } func (r listSoftwareTitlesResponse) error() error { return r.Err } @@ -44,7 +44,7 @@ func listSoftwareTitlesEndpoint(ctx context.Context, request interface{}, svc fl } } if len(titles) == 0 { - titles = []fleet.SoftwareTitle{} + titles = []fleet.SoftwareTitleListResult{} } listResp := listSoftwareTitlesResponse{ SoftwareTitles: titles, @@ -61,7 +61,7 @@ func listSoftwareTitlesEndpoint(ctx context.Context, request interface{}, svc fl func (svc *Service) ListSoftwareTitles( ctx context.Context, opt fleet.SoftwareTitleListOptions, -) ([]fleet.SoftwareTitle, int, *fleet.PaginationMetadata, error) { +) ([]fleet.SoftwareTitleListResult, int, *fleet.PaginationMetadata, error) { if err := svc.authz.Authorize(ctx, &fleet.AuthzSoftwareInventory{ TeamID: opt.TeamID, }, fleet.ActionRead); err != nil { diff --git a/server/service/software_titles_test.go b/server/service/software_titles_test.go index 204b8434eb..599cba1774 100644 --- a/server/service/software_titles_test.go +++ b/server/service/software_titles_test.go @@ -15,8 +15,8 @@ import ( func TestServiceSoftwareTitlesAuth(t *testing.T) { ds := new(mock.Store) - ds.ListSoftwareTitlesFunc = func(ctx context.Context, opt fleet.SoftwareTitleListOptions, tmf fleet.TeamFilter) ([]fleet.SoftwareTitle, int, *fleet.PaginationMetadata, error) { - return []fleet.SoftwareTitle{}, 0, &fleet.PaginationMetadata{}, nil + ds.ListSoftwareTitlesFunc = func(ctx context.Context, opt fleet.SoftwareTitleListOptions, tmf fleet.TeamFilter) ([]fleet.SoftwareTitleListResult, int, *fleet.PaginationMetadata, error) { + return []fleet.SoftwareTitleListResult{}, 0, &fleet.PaginationMetadata{}, nil } ds.SoftwareTitleByIDFunc = func(ctx context.Context, id uint, teamID *uint, tmFilter fleet.TeamFilter) (*fleet.SoftwareTitle, error) { return &fleet.SoftwareTitle{}, nil From 48684fb6cab8ad0cd300e4e0d6fc35e033d9d819 Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Wed, 15 May 2024 12:43:12 -0500 Subject: [PATCH 49/56] Fix unreleased issues in software installers UI: Part 5 (#19033) --- .../InstallStatusCell/InstallStatusCell.tsx | 34 +++++++++++++------ .../Software/InstallStatusCell/_styles.scss | 4 +++ .../hosts/details/cards/Software/Software.tsx | 10 ++++-- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/frontend/pages/hosts/details/cards/Software/InstallStatusCell/InstallStatusCell.tsx b/frontend/pages/hosts/details/cards/Software/InstallStatusCell/InstallStatusCell.tsx index 7ea5fb3495..2d7112566e 100644 --- a/frontend/pages/hosts/details/cards/Software/InstallStatusCell/InstallStatusCell.tsx +++ b/frontend/pages/hosts/details/cards/Software/InstallStatusCell/InstallStatusCell.tsx @@ -1,10 +1,12 @@ import React, { ReactNode } from "react"; +import ReactTooltip from "react-tooltip"; +import { uniqueId } from "lodash"; + import { SoftwareInstallStatus } from "interfaces/software"; import { dateAgo } from "utilities/date_format"; import Icon from "components/Icon"; -import TooltipWrapper from "components/TooltipWrapper"; import TextCell from "components/TableContainer/DataTable/TextCell"; const baseClass = "install-status-cell"; @@ -77,20 +79,30 @@ const InstallStatusCell = ({ } const displayConfig = CELL_DISPLAY_OPTIONS[displayStatus]; + const tooltipId = uniqueId(); return ( - -
+
+
- {displayConfig.displayText}
- + + + {displayConfig.tooltip(packageToInstall, installedAt)} + + + {displayConfig.displayText} +
); }; diff --git a/frontend/pages/hosts/details/cards/Software/InstallStatusCell/_styles.scss b/frontend/pages/hosts/details/cards/Software/InstallStatusCell/_styles.scss index 10f8b9c53a..7384269a1e 100644 --- a/frontend/pages/hosts/details/cards/Software/InstallStatusCell/_styles.scss +++ b/frontend/pages/hosts/details/cards/Software/InstallStatusCell/_styles.scss @@ -8,4 +8,8 @@ &__status-tooltip { text-align: center; } + + &__status-tooltip-text { + font-size: $xx-small; + } } diff --git a/frontend/pages/hosts/details/cards/Software/Software.tsx b/frontend/pages/hosts/details/cards/Software/Software.tsx index 0ce821199e..9a9476deee 100644 --- a/frontend/pages/hosts/details/cards/Software/Software.tsx +++ b/frontend/pages/hosts/details/cards/Software/Software.tsx @@ -2,6 +2,7 @@ import React, { useCallback, useContext, useMemo, useState } from "react"; import { InjectedRouter } from "react-router"; import { useQuery } from "react-query"; import { AxiosError } from "axios"; +import { trimEnd, upperFirst } from "lodash"; import hostAPI, { IGetHostSoftwareResponse, @@ -166,9 +167,14 @@ const SoftwareCard = ({ "Software is installing or will install when the host comes online." ); } catch (e) { - const reason = getErrorReason(e); + const reason = upperFirst(trimEnd(getErrorReason(e), ".")); if (reason.includes("fleetd installed")) { - renderFlash("error", reason); + renderFlash("error", `Couldn't install. ${reason}.`); + } else if (reason.includes("can be installed only on")) { + renderFlash( + "error", + `Couldn't install. ${reason.replace("darwin", "macOS")}.` + ); } else { renderFlash("error", "Couldn't install. Please try again."); } From 1adb7677fd91663373d7039891d747be3a877bc2 Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Wed, 15 May 2024 14:32:20 -0500 Subject: [PATCH 50/56] Validate installer file size in UI; add install tooltip to software list (#19036) --- .../SoftwareNameCell/SoftwareNameCell.tsx | 34 +++++++++++++++++-- .../DataTable/SoftwareNameCell/_styles.scss | 5 +++ .../AddSoftwareModal/AddSoftwareModal.tsx | 12 +++++++ 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/frontend/components/TableContainer/DataTable/SoftwareNameCell/SoftwareNameCell.tsx b/frontend/components/TableContainer/DataTable/SoftwareNameCell/SoftwareNameCell.tsx index 7f87581b55..47c5a97a58 100644 --- a/frontend/components/TableContainer/DataTable/SoftwareNameCell/SoftwareNameCell.tsx +++ b/frontend/components/TableContainer/DataTable/SoftwareNameCell/SoftwareNameCell.tsx @@ -1,5 +1,8 @@ import React from "react"; import { InjectedRouter } from "react-router"; +import ReactTooltip from "react-tooltip"; + +import { uniqueId } from "lodash"; import Icon from "components/Icon"; @@ -9,6 +12,33 @@ import LinkCell from "../LinkCell"; const baseClass = "software-name-cell"; +const InstallIconWithTooltip = () => { + const tooltipId = uniqueId(); + return ( +
+
+ +
+ + + Software can be installed on Host details page. + + +
+ ); +}; + interface ISoftwareNameCellProps { name: string; source: string; @@ -50,9 +80,7 @@ const SoftwareNameCell = ({ <> {name} - {hasPackage && ( - - )} + {hasPackage && } } /> diff --git a/frontend/components/TableContainer/DataTable/SoftwareNameCell/_styles.scss b/frontend/components/TableContainer/DataTable/SoftwareNameCell/_styles.scss index d387075dd1..23a3f5a6a0 100644 --- a/frontend/components/TableContainer/DataTable/SoftwareNameCell/_styles.scss +++ b/frontend/components/TableContainer/DataTable/SoftwareNameCell/_styles.scss @@ -19,4 +19,9 @@ // the same issue as the .software-name-cell class display value. display: inline-flex !important; } + + &__install-tooltip-text { + font-weight: $regular; + font-size: $xx-small; + } } diff --git a/frontend/pages/SoftwarePage/components/AddSoftwareModal/AddSoftwareModal.tsx b/frontend/pages/SoftwarePage/components/AddSoftwareModal/AddSoftwareModal.tsx index bc2c268471..4391c68f4b 100644 --- a/frontend/pages/SoftwarePage/components/AddSoftwareModal/AddSoftwareModal.tsx +++ b/frontend/pages/SoftwarePage/components/AddSoftwareModal/AddSoftwareModal.tsx @@ -16,6 +16,8 @@ import { IAddSoftwareFormData } from "../AddSoftwareForm/AddSoftwareForm"; // 2 minutes const UPLOAD_TIMEOUT = 120000; +const MAX_FILE_SIZE_MB = 500; +const MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024; const baseClass = "add-software-modal"; @@ -83,6 +85,16 @@ const AddSoftwareModal = ({ const onAddSoftware = async (formData: IAddSoftwareFormData) => { setIsUploading(true); + if (formData.software && formData.software.size > MAX_FILE_SIZE_BYTES) { + renderFlash( + "error", + `Couldn’t add. The maximum file size is ${MAX_FILE_SIZE_MB} MB.` + ); + onExit(); + setIsUploading(false); + return; + } + try { await softwareAPI.addSoftwarePackage(formData, teamId); renderFlash( From 01898fd176677ce6acad5461bd908901a937fe11 Mon Sep 17 00:00:00 2001 From: Jahziel Villasana-Espinoza Date: Wed, 15 May 2024 16:36:31 -0400 Subject: [PATCH 51/56] fix: typos in scripts (#19045) Feature cleanup # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [x] Manual QA for all new/changed functionality --- changes/jve-fix-script-typo | 1 + pkg/file/scripts/install_msi.ps1 | 2 +- pkg/file/scripts/remove_msi.ps1 | 2 +- pkg/file/testdata/scripts/install_msi.ps1.golden | 2 +- pkg/file/testdata/scripts/remove_msi.ps1.golden | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changes/jve-fix-script-typo diff --git a/changes/jve-fix-script-typo b/changes/jve-fix-script-typo new file mode 100644 index 0000000000..8f4ab4fbe4 --- /dev/null +++ b/changes/jve-fix-script-typo @@ -0,0 +1 @@ +- Fixes some typos that were in the Powershell scripts for installing Windows software. \ No newline at end of file diff --git a/pkg/file/scripts/install_msi.ps1 b/pkg/file/scripts/install_msi.ps1 index e4f8d8ca90..0f7595481d 100644 --- a/pkg/file/scripts/install_msi.ps1 +++ b/pkg/file/scripts/install_msi.ps1 @@ -6,4 +6,4 @@ $installProcess = Start-Process msiexec.exe ` Get-Content $logFile -Tail 500 -exit $instalProcess.ExitCode +exit $installProcess.ExitCode diff --git a/pkg/file/scripts/remove_msi.ps1 b/pkg/file/scripts/remove_msi.ps1 index 899a4c9bec..23172920ac 100644 --- a/pkg/file/scripts/remove_msi.ps1 +++ b/pkg/file/scripts/remove_msi.ps1 @@ -6,4 +6,4 @@ $removeProcess = Start-Process msiexec.exe ` Get-Content $logFile -Tail 500 -exit $instalProcess.ExitCode +exit $removeProcess.ExitCode diff --git a/pkg/file/testdata/scripts/install_msi.ps1.golden b/pkg/file/testdata/scripts/install_msi.ps1.golden index e4f8d8ca90..0f7595481d 100644 --- a/pkg/file/testdata/scripts/install_msi.ps1.golden +++ b/pkg/file/testdata/scripts/install_msi.ps1.golden @@ -6,4 +6,4 @@ $installProcess = Start-Process msiexec.exe ` Get-Content $logFile -Tail 500 -exit $instalProcess.ExitCode +exit $installProcess.ExitCode diff --git a/pkg/file/testdata/scripts/remove_msi.ps1.golden b/pkg/file/testdata/scripts/remove_msi.ps1.golden index 899a4c9bec..23172920ac 100644 --- a/pkg/file/testdata/scripts/remove_msi.ps1.golden +++ b/pkg/file/testdata/scripts/remove_msi.ps1.golden @@ -6,4 +6,4 @@ $removeProcess = Start-Process msiexec.exe ` Get-Content $logFile -Tail 500 -exit $instalProcess.ExitCode +exit $removeProcess.ExitCode From 62adb46a367cd47bdd6be5e24074722bfbff6efc Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 15 May 2024 17:27:04 -0400 Subject: [PATCH 52/56] Partial fix: fix empty software title when metadata doesn't find one (#19047) #19041 (it falls back on filename) # Checklist for submitter - [x] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements) --- ee/server/service/software_installers.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index eb617ccffa..2f8f8271af 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -321,6 +321,10 @@ func (svc *Service) addMetadataToSoftwarePayload(ctx context.Context, payload *f } return "", ctxerr.Wrap(ctx, err, "extracting metadata from installer") } + if title == "" { + // use the filename if no title from metadata + title = payload.Filename + } payload.Title = title payload.Version = vers payload.StorageID = hex.EncodeToString(hash) @@ -445,11 +449,7 @@ func (svc *Service) BatchSetSoftwareInstallers(ctx context.Context, tmName strin InstallerFile: bytes.NewReader(bodyBytes), } - ext, err := svc.addMetadataToSoftwarePayload(ctx, installer) - if err != nil { - return err - } - + // set the filename before adding metadata, as it is used as fallback var filename string cdh, ok := resp.Header["Content-Disposition"] if ok && len(cdh) > 0 { @@ -458,7 +458,15 @@ func (svc *Service) BatchSetSoftwareInstallers(ctx context.Context, tmName strin filename = params["filename"] } } - // if it fails, try to extract it from the URL + installer.Filename = filename + + ext, err := svc.addMetadataToSoftwarePayload(ctx, installer) + if err != nil { + return err + } + + // if filename was empty, try to extract it from the URL with the + // now-known extension if filename == "" { filename = file.ExtractFilenameFromURLPath(p.URL, ext) } @@ -467,6 +475,9 @@ func (svc *Service) BatchSetSoftwareInstallers(ctx context.Context, tmName strin filename = fmt.Sprintf("package.%s", ext) } installer.Filename = filename + if installer.Title == "" { + installer.Title = filename + } installers[i] = installer From ad94dff814813512e8453e375966b5a489b96789 Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Wed, 15 May 2024 19:18:35 -0300 Subject: [PATCH 53/56] installer report and rollback fixes (#19046) for https://github.com/fleetdm/fleet/issues/19020 - Fixes the rollback logic to get the right script for the software being installed - Fixes the messages displayed in the install results # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Added/updated tests - [x] Manual QA for all new/changed functionality --- orbit/pkg/installer/installer.go | 15 +- orbit/pkg/installer/installer_test.go | 28 ++++ pkg/file/scripts/install_deb.sh | 2 +- .../testdata/scripts/install_deb.sh.golden | 2 +- server/datastore/mysql/software_installers.go | 6 +- .../mysql/software_installers_test.go | 19 +-- server/fleet/software_installer.go | 32 +++-- server/fleet/software_test.go | 130 +++++++++++------- server/service/integration_enterprise_test.go | 18 +-- 9 files changed, 160 insertions(+), 92 deletions(-) diff --git a/orbit/pkg/installer/installer.go b/orbit/pkg/installer/installer.go index 4109dcea61..669eb38c22 100644 --- a/orbit/pkg/installer/installer.go +++ b/orbit/pkg/installer/installer.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "sync" "time" @@ -105,6 +106,7 @@ func connectOsquery(r *Runner) error { } func (r *Runner) run(ctx context.Context, config *fleet.OrbitConfig) error { + log.Debug().Msg("starting software installers run") var errs []error for _, installerID := range config.Notifications.PendingSoftwareInstallerIDs { if ctx.Err() != nil { @@ -160,6 +162,7 @@ func (r *Runner) preConditionCheck(ctx context.Context, query string) (bool, str } func (r *Runner) installSoftware(ctx context.Context, installID string) (*fleet.HostSoftwareInstallResultPayload, error) { + log.Debug().Msgf("about to install software with installer id: %s", installID) installer, err := r.OrbitClient.GetInstallerDetails(installID) if err != nil { return nil, fmt.Errorf("fetching software installer details: %w", err) @@ -169,6 +172,7 @@ func (r *Runner) installSoftware(ctx context.Context, installID string) (*fleet. payload.InstallUUID = installID if installer.PreInstallCondition != "" { + log.Debug().Msgf("pre-condition is not empty, about to run the query") shouldInstall, output, err := r.preConditionCheck(ctx, installer.PreInstallCondition) payload.PreInstallConditionOutput = &output if err != nil { @@ -176,12 +180,14 @@ func (r *Runner) installSoftware(ctx context.Context, installID string) (*fleet. } if !shouldInstall { + log.Debug().Msgf("pre-condition didn't pass, stopping installation") return payload, nil } } if !r.scriptsEnabled() { // fleetctl knows that -2 means script was disabled on host + log.Debug().Msgf("scripts are disabled for this host, stopping installation") payload.InstallScriptExitCode = ptr.Int(-2) payload.InstallScriptOutput = ptr.String("Scripts are disabled") return payload, nil @@ -196,6 +202,7 @@ func (r *Runner) installSoftware(ctx context.Context, installID string) (*fleet. return payload, fmt.Errorf("creating temporary directory: %w", err) } + log.Debug().Msgf("about to download software installer") installerPath, err := r.OrbitClient.DownloadSoftwareInstaller(installer.InstallerID, tmpDir) if err != nil { return payload, err @@ -217,6 +224,7 @@ func (r *Runner) installSoftware(ctx context.Context, installID string) (*fleet. if runtime.GOOS == "windows" { scriptExtension = ".ps1" } + log.Debug().Msgf("about to run install script") installOutput, installExitCode, err := r.runInstallerScript(ctx, installer.InstallScript, installerPath, "install-script"+scriptExtension) payload.InstallScriptOutput = &installOutput payload.InstallScriptExitCode = &installExitCode @@ -225,20 +233,23 @@ func (r *Runner) installSoftware(ctx context.Context, installID string) (*fleet. } if installer.PostInstallScript != "" { + log.Debug().Msgf("about to run post-install script") postOutput, postExitCode, postErr := r.runInstallerScript(ctx, installer.PostInstallScript, installerPath, "post-install-script"+scriptExtension) payload.PostInstallScriptOutput = &postOutput payload.PostInstallScriptExitCode = &postExitCode if postErr != nil || postExitCode != 0 { log.Info().Msgf("installation of %s failed, attempting rollback. Exit code: %d, error: %s", installerPath, postExitCode, postErr) - uninstallScript := file.GetRemoveScript(installerPath) + ext := filepath.Ext(installerPath) + ext = strings.TrimPrefix(ext, ".") + uninstallScript := file.GetRemoveScript(ext) uninstallOutput, uninstallExitCode, uninstallErr := r.runInstallerScript(ctx, uninstallScript, installerPath, "rollback-script") log.Info().Msgf( "rollback staus: exit code: %d, error: %s, output: %s", uninstallExitCode, uninstallErr, uninstallOutput, ) - return payload, postErr + return payload, uninstallErr } } diff --git a/orbit/pkg/installer/installer_test.go b/orbit/pkg/installer/installer_test.go index 7b75a7adbb..58d5eda98d 100644 --- a/orbit/pkg/installer/installer_test.go +++ b/orbit/pkg/installer/installer_test.go @@ -363,6 +363,34 @@ func TestInstallerRun(t *testing.T) { return execOutput, execExitCode, execErr } + err := r.run(context.Background(), &config) + require.NoError(t, err) + + require.True(t, downloadInstallerFnCalled) + require.True(t, execCalled) + require.True(t, removeAllFnCalled) + require.NotNil(t, savedInstallerResult) + require.Equal(t, installDetails.ExecutionID, savedInstallerResult.InstallUUID) + require.Equal(t, 0, *savedInstallerResult.InstallScriptExitCode) + require.Equal(t, string(execOutput), *savedInstallerResult.InstallScriptOutput) + require.Equal(t, 1, *savedInstallerResult.PostInstallScriptExitCode) + require.Equal(t, string(execOutput), *savedInstallerResult.PostInstallScriptOutput) + }) + + t.Run("failed rollback script", func(t *testing.T) { + resetAll() + + r.execCmdFn = func(ctx context.Context, scriptPath string, env []string) ([]byte, int, error) { + execCalled = true + execEnv = env + executedScripts = append(executedScripts, scriptPath) + // bad exit on the post-install and rollback script + if len(executedScripts) >= 2 { + return execOutput, 1, &exec.ExitError{} + } + return execOutput, execExitCode, execErr + } + err := r.run(context.Background(), &config) require.Error(t, err) diff --git a/pkg/file/scripts/install_deb.sh b/pkg/file/scripts/install_deb.sh index 78dbeafbe3..a12695f586 100644 --- a/pkg/file/scripts/install_deb.sh +++ b/pkg/file/scripts/install_deb.sh @@ -1,3 +1,3 @@ #!/bin/sh -apt-get install -f "$INSTALLER_PATH" +apt-get install --assume-yes -f "$INSTALLER_PATH" diff --git a/pkg/file/testdata/scripts/install_deb.sh.golden b/pkg/file/testdata/scripts/install_deb.sh.golden index 78dbeafbe3..a12695f586 100644 --- a/pkg/file/testdata/scripts/install_deb.sh.golden +++ b/pkg/file/testdata/scripts/install_deb.sh.golden @@ -1,3 +1,3 @@ #!/bin/sh -apt-get install -f "$INSTALLER_PATH" +apt-get install --assume-yes -f "$INSTALLER_PATH" diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 17588a13c4..a19a4b35bc 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -285,9 +285,9 @@ func (ds *Datastore) GetSoftwareInstallResults(ctx context.Context, resultsUUID query := fmt.Sprintf(` SELECT hsi.execution_id AS execution_id, - COALESCE(hsi.pre_install_query_output, '') AS pre_install_query_output, - COALESCE(hsi.post_install_script_output, '') AS post_install_script_output, - COALESCE(hsi.install_script_output, '') AS install_script_output, + hsi.pre_install_query_output, + hsi.post_install_script_output, + hsi.install_script_output, hsi.host_id AS host_id, h.computer_name AS host_display_name, st.name AS software_title, diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go index 7eb58f8f65..4fd6e7e5d5 100644 --- a/server/datastore/mysql/software_installers_test.go +++ b/server/datastore/mysql/software_installers_test.go @@ -291,22 +291,9 @@ func testGetSoftwareInstallResult(t *testing.T, ds *Datastore) { require.Equal(t, swFilename, res.SoftwarePackage) require.Equal(t, host.ID, res.HostID) require.Equal(t, host.DisplayName(), res.HostDisplayName) - expectedPreInstallQueryOutput := "" - if tc.preInstallQueryOutput != nil { - expectedPreInstallQueryOutput = *tc.preInstallQueryOutput - } - require.Equal(t, expectedPreInstallQueryOutput, res.PreInstallQueryOutput) - - expectedPostInstallScriptOutput := "" - if tc.postInstallScriptOutput != nil { - expectedPostInstallScriptOutput = *tc.postInstallScriptOutput - } - require.Equal(t, expectedPostInstallScriptOutput, res.PostInstallScriptOutput) - expectedInstallScriptOutput := "" - if tc.installScriptOutput != nil { - expectedInstallScriptOutput = *tc.installScriptOutput - } - require.Equal(t, expectedInstallScriptOutput, res.Output) + require.Equal(t, tc.preInstallQueryOutput, res.PreInstallQueryOutput) + require.Equal(t, tc.postInstallScriptOutput, res.PostInstallScriptOutput) + require.Equal(t, tc.installScriptOutput, res.Output) }) } } diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index 1e8f47693b..517a434a92 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -6,6 +6,8 @@ import ( "fmt" "io" "time" + + "github.com/fleetdm/fleet/v4/server/ptr" ) // SoftwareInstallerStore is the interface to store and retrieve software @@ -154,11 +156,11 @@ type HostSoftwareInstallerResult struct { // have specific values that should be used? If so, how are they calculated? Detail string `json:"detail" db:"detail"` // Output is the output of the software installer package on the host. - Output string `json:"output" db:"install_script_output"` + Output *string `json:"output" db:"install_script_output"` // PreInstallQueryOutput is the output of the pre-install query on the host. - PreInstallQueryOutput string `json:"pre_install_query_output" db:"pre_install_query_output"` + PreInstallQueryOutput *string `json:"pre_install_query_output" db:"pre_install_query_output"` // PostInstallScriptOutput is the output of the post-install script on the host. - PostInstallScriptOutput string `json:"post_install_script_output" db:"post_install_script_output"` + PostInstallScriptOutput *string `json:"post_install_script_output" db:"post_install_script_output"` // CreatedAt is the time the software installer request was triggered. CreatedAt time.Time `json:"created_at" db:"created_at"` // UpdatedAt is the time the software installer request was last updated. @@ -197,33 +199,35 @@ func (h *HostSoftwareInstallerResult) EnhanceOutputDetails() { return } - if h.PreInstallQueryOutput == "" { - h.PreInstallQueryOutput = SoftwareInstallerQueryFailCopy - return + if h.PreInstallQueryOutput != nil { + if *h.PreInstallQueryOutput == "" { + *h.PreInstallQueryOutput = SoftwareInstallerQueryFailCopy + return + } + *h.PreInstallQueryOutput = SoftwareInstallerQuerySuccessCopy } - h.PreInstallQueryOutput = SoftwareInstallerQuerySuccessCopy - if h.InstallScriptExitCode == nil { + if h.Output == nil || h.InstallScriptExitCode == nil { return } if *h.InstallScriptExitCode == -2 { - h.Output = SoftwareInstallerScriptsDisabledCopy + *h.Output = SoftwareInstallerScriptsDisabledCopy return } else if *h.InstallScriptExitCode != 0 { - h.Output = fmt.Sprintf(SoftwareInstallerInstallFailCopy, h.Output) + h.Output = ptr.String(fmt.Sprintf(SoftwareInstallerInstallFailCopy, *h.Output)) return } - h.Output = fmt.Sprintf(SoftwareInstallerInstallSuccessCopy, h.Output) + h.Output = ptr.String(fmt.Sprintf(SoftwareInstallerInstallSuccessCopy, *h.Output)) - if h.PostInstallScriptExitCode == nil { + if h.PostInstallScriptExitCode == nil || h.PostInstallScriptOutput == nil { return } if *h.PostInstallScriptExitCode != 0 { - h.PostInstallScriptOutput = fmt.Sprintf(SoftwareInstallerPostInstallFailCopy, *h.PostInstallScriptExitCode, h.PostInstallScriptOutput) + h.PostInstallScriptOutput = ptr.String(fmt.Sprintf(SoftwareInstallerPostInstallFailCopy, *h.PostInstallScriptExitCode, *h.PostInstallScriptOutput)) return } - h.PostInstallScriptOutput = fmt.Sprintf(SoftwareInstallerPostInstallSuccessCopy, h.PostInstallScriptOutput) + h.PostInstallScriptOutput = ptr.String(fmt.Sprintf(SoftwareInstallerPostInstallSuccessCopy, *h.PostInstallScriptOutput)) } type HostSoftwareInstallerResultAuthz struct { diff --git a/server/fleet/software_test.go b/server/fleet/software_test.go index c00708c0f7..236eb896a3 100644 --- a/server/fleet/software_test.go +++ b/server/fleet/software_test.go @@ -1,6 +1,7 @@ package fleet import ( + "fmt" "testing" "github.com/fleetdm/fleet/v4/server/ptr" @@ -80,79 +81,116 @@ func TestParseSoftwareLastOpenedAtRowValue(t *testing.T) { func TestEnhanceOutputDetails(t *testing.T) { tests := []struct { name string - hsr HostSoftwareInstallerResult - expectedPreInstallQueryOutput string - expectedOutput string - expectedPostInstallScriptOutput string + initial HostSoftwareInstallerResult + expectedPreInstallQueryOutput *string + expectedOutput *string + expectedPostInstallScriptOutput *string }{ { name: "pending status", - hsr: HostSoftwareInstallerResult{ + initial: HostSoftwareInstallerResult{ Status: SoftwareInstallerPending, }, - expectedPreInstallQueryOutput: "", - expectedOutput: "", - expectedPostInstallScriptOutput: "", + expectedPreInstallQueryOutput: nil, + expectedOutput: nil, + expectedPostInstallScriptOutput: nil, }, { - name: "non-pending status with empty PreInstallQueryOutput and successful install", - hsr: HostSoftwareInstallerResult{ + name: "non-pending status with empty PreInstallQueryOutput", + initial: HostSoftwareInstallerResult{ Status: SoftwareInstallerInstalled, - InstallScriptExitCode: ptr.Int(0), - PreInstallQueryOutput: "1", + PreInstallQueryOutput: ptr.String(""), }, - expectedPreInstallQueryOutput: "Query returned result\nProceeding to install...", - expectedOutput: "Installing software...\nSuccess\n", - expectedPostInstallScriptOutput: "", + expectedPreInstallQueryOutput: ptr.String(SoftwareInstallerQueryFailCopy), + expectedOutput: nil, + expectedPostInstallScriptOutput: nil, }, { - name: "non-pending status with empty PreInstallQueryOutput and failed install", - hsr: HostSoftwareInstallerResult{ + name: "non-pending status with non-empty PreInstallQueryOutput", + initial: HostSoftwareInstallerResult{ + Status: SoftwareInstallerInstalled, + PreInstallQueryOutput: ptr.String("Some output"), + }, + expectedPreInstallQueryOutput: ptr.String(SoftwareInstallerQuerySuccessCopy), + expectedOutput: nil, + expectedPostInstallScriptOutput: nil, + }, + { + name: "non-pending status with nil PreInstallQueryOutput", + initial: HostSoftwareInstallerResult{ + Status: SoftwareInstallerInstalled, + }, + expectedPreInstallQueryOutput: nil, + expectedOutput: nil, + expectedPostInstallScriptOutput: nil, + }, + { + name: "non-pending status with install scripts disabled", + initial: HostSoftwareInstallerResult{ + Status: SoftwareInstallerInstalled, + InstallScriptExitCode: ptr.Int(-2), + Output: ptr.String(""), + }, + expectedPreInstallQueryOutput: nil, + expectedOutput: ptr.String(SoftwareInstallerScriptsDisabledCopy), + expectedPostInstallScriptOutput: nil, + }, + { + name: "non-pending status with failed install script", + initial: HostSoftwareInstallerResult{ Status: SoftwareInstallerFailed, InstallScriptExitCode: ptr.Int(1), - PreInstallQueryOutput: "1", + Output: ptr.String("Some install output"), }, - expectedPreInstallQueryOutput: "Query returned result\nProceeding to install...", - expectedOutput: "Installing software...\nFailed\n", - expectedPostInstallScriptOutput: "", + expectedPreInstallQueryOutput: nil, + expectedOutput: ptr.String(fmt.Sprintf(SoftwareInstallerInstallFailCopy, "Some install output")), + expectedPostInstallScriptOutput: nil, }, { - name: "non-pending status with non-empty PreInstallQueryOutput and disabled scripts", - hsr: HostSoftwareInstallerResult{ - Status: SoftwareInstallerFailed, - InstallScriptExitCode: ptr.Int(-2), - PreInstallQueryOutput: "1", + name: "non-pending status with successful install script", + initial: HostSoftwareInstallerResult{ + Status: SoftwareInstallerInstalled, + InstallScriptExitCode: ptr.Int(0), + Output: ptr.String("Some install output"), }, - expectedPreInstallQueryOutput: "Query returned result\nProceeding to install...", - expectedOutput: "Installing software...\nError: Scripts are disabled for this host. To run scripts, deploy the fleetd agent with --scripts-enabled.", - expectedPostInstallScriptOutput: "", + expectedPreInstallQueryOutput: nil, + expectedOutput: ptr.String(fmt.Sprintf(SoftwareInstallerInstallSuccessCopy, "Some install output")), + expectedPostInstallScriptOutput: nil, }, { - name: "non-pending status with non-empty PreInstallQueryOutput and failed post install script", - hsr: HostSoftwareInstallerResult{ + name: "non-pending status with successful post install script", + initial: HostSoftwareInstallerResult{ Status: SoftwareInstallerInstalled, InstallScriptExitCode: ptr.Int(0), - PostInstallScriptExitCode: ptr.Int(1), - PreInstallQueryOutput: "1", - PostInstallScriptOutput: "output!", + Output: ptr.String("Some install output"), + PostInstallScriptExitCode: ptr.Int(0), + PostInstallScriptOutput: ptr.String("Some post install output"), }, - expectedPreInstallQueryOutput: "Query returned result\nProceeding to install...", - expectedOutput: "Installing software...\nSuccess\n", - expectedPostInstallScriptOutput: `Running script... -Exit code: 1 (Failed) -output! -Rolling back software install... -Rolled back successfully -`, + expectedPreInstallQueryOutput: nil, + expectedOutput: ptr.String(fmt.Sprintf(SoftwareInstallerInstallSuccessCopy, "Some install output")), + expectedPostInstallScriptOutput: ptr.String(fmt.Sprintf(SoftwareInstallerPostInstallSuccessCopy, "Some post install output")), + }, + { + name: "non-pending status with failed post install script", + initial: HostSoftwareInstallerResult{ + Status: SoftwareInstallerInstalled, + InstallScriptExitCode: ptr.Int(0), + Output: ptr.String("Some install output"), + PostInstallScriptExitCode: ptr.Int(1), + PostInstallScriptOutput: ptr.String("Some post install output"), + }, + expectedPreInstallQueryOutput: nil, + expectedOutput: ptr.String(fmt.Sprintf(SoftwareInstallerInstallSuccessCopy, "Some install output")), + expectedPostInstallScriptOutput: ptr.String(fmt.Sprintf(SoftwareInstallerPostInstallFailCopy, 1, "Some post install output")), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - tt.hsr.EnhanceOutputDetails() - require.Equal(t, tt.expectedPreInstallQueryOutput, tt.hsr.PreInstallQueryOutput) - require.Equal(t, tt.expectedOutput, tt.hsr.Output) - require.Equal(t, tt.expectedPostInstallScriptOutput, tt.hsr.PostInstallScriptOutput) + tt.initial.EnhanceOutputDetails() + require.Equal(t, tt.expectedPreInstallQueryOutput, tt.initial.PreInstallQueryOutput) + require.Equal(t, tt.expectedOutput, tt.initial.Output) + require.Equal(t, tt.expectedPostInstallScriptOutput, tt.initial.PostInstallScriptOutput) }) } } diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 32325d5a57..779ee84599 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -9562,9 +9562,9 @@ func (s *integrationEnterpriseTestSuite) TestHostSoftwareInstallResult() { HostID uint InstallUUID string Status fleet.SoftwareInstallerStatus - Output string - PostInstallScriptOutput string - PreInstallQueryOutput string + Output *string + PostInstallScriptOutput *string + PreInstallQueryOutput *string } checkResults := func(want result) { var resp getSoftwareInstallResultsResponse @@ -9591,8 +9591,8 @@ func (s *integrationEnterpriseTestSuite) TestHostSoftwareInstallResult() { HostID: host.ID, InstallUUID: installUUIDs[0], Status: fleet.SoftwareInstallerFailed, - PreInstallQueryOutput: fleet.SoftwareInstallerQuerySuccessCopy, - Output: fmt.Sprintf(fleet.SoftwareInstallerInstallFailCopy, "failed"), + PreInstallQueryOutput: ptr.String(fleet.SoftwareInstallerQuerySuccessCopy), + Output: ptr.String(fmt.Sprintf(fleet.SoftwareInstallerInstallFailCopy, "failed")), }) wantAct := fleet.ActivityTypeInstalledSoftware{ HostID: host.ID, @@ -9614,7 +9614,7 @@ func (s *integrationEnterpriseTestSuite) TestHostSoftwareInstallResult() { HostID: host.ID, InstallUUID: installUUIDs[1], Status: fleet.SoftwareInstallerFailed, - PreInstallQueryOutput: fleet.SoftwareInstallerQueryFailCopy, + PreInstallQueryOutput: ptr.String(fleet.SoftwareInstallerQueryFailCopy), }) wantAct.InstallUUID = installUUIDs[1] s.lastActivityOfTypeMatches(wantAct.ActivityName(), string(jsonMustMarshal(t, wantAct)), 0) @@ -9634,9 +9634,9 @@ func (s *integrationEnterpriseTestSuite) TestHostSoftwareInstallResult() { HostID: host.ID, InstallUUID: installUUIDs[2], Status: fleet.SoftwareInstallerInstalled, - PreInstallQueryOutput: fleet.SoftwareInstallerQuerySuccessCopy, - Output: fmt.Sprintf(fleet.SoftwareInstallerInstallSuccessCopy, "success"), - PostInstallScriptOutput: fmt.Sprintf(fleet.SoftwareInstallerPostInstallSuccessCopy, "ok"), + PreInstallQueryOutput: ptr.String(fleet.SoftwareInstallerQuerySuccessCopy), + Output: ptr.String(fmt.Sprintf(fleet.SoftwareInstallerInstallSuccessCopy, "success")), + PostInstallScriptOutput: ptr.String(fmt.Sprintf(fleet.SoftwareInstallerPostInstallSuccessCopy, "ok")), }) wantAct.InstallUUID = installUUIDs[2] wantAct.Status = string(fleet.SoftwareInstallerInstalled) From d383876a3c8beaf88a353d4eafa63f342cc628cd Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Wed, 15 May 2024 19:39:42 -0300 Subject: [PATCH 54/56] fix issues installing software in windows (#19048) for #19039 and #19041 this: - fixes the install/remove scripts to read the env variable the proper way - truncates output before storing in the databse in case its longer than MySQL's TEXT size # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Added/updated tests - [x] Manual QA for all new/changed functionality --- pkg/file/scripts/install_exe.ps1 | 2 +- pkg/file/scripts/install_msi.ps1 | 2 +- pkg/file/scripts/remove_exe.ps1 | 2 +- pkg/file/scripts/remove_msi.ps1 | 2 +- .../testdata/scripts/install_exe.ps1.golden | 2 +- .../testdata/scripts/install_msi.ps1.golden | 2 +- .../testdata/scripts/remove_exe.ps1.golden | 2 +- .../testdata/scripts/remove_msi.ps1.golden | 2 +- server/datastore/mysql/scripts.go | 26 +++++++++++-------- server/datastore/mysql/software.go | 15 ++++++++--- 10 files changed, 35 insertions(+), 22 deletions(-) diff --git a/pkg/file/scripts/install_exe.ps1 b/pkg/file/scripts/install_exe.ps1 index e205c4ed11..f9d762b318 100644 --- a/pkg/file/scripts/install_exe.ps1 +++ b/pkg/file/scripts/install_exe.ps1 @@ -1,4 +1,4 @@ -$exeFilePath = "$INSTALLER_PATH" +$exeFilePath = "${env:INSTALLER_PATH}" # extract the name of the executable to use as the sub-directory name $exeName = [System.IO.Path]::GetFileName($exeFilePath) diff --git a/pkg/file/scripts/install_msi.ps1 b/pkg/file/scripts/install_msi.ps1 index 0f7595481d..838c431c1d 100644 --- a/pkg/file/scripts/install_msi.ps1 +++ b/pkg/file/scripts/install_msi.ps1 @@ -1,7 +1,7 @@ $logFile = "${env:TEMP}/fleet-install-software.log" $installProcess = Start-Process msiexec.exe ` - -ArgumentList "/quiet /norestart /lv ${logFile} /i `"$INSTALLER_PATH`"" ` + -ArgumentList "/quiet /norestart /lv ${logFile} /i `"${env:INSTALLER_PATH}`"" ` -PassThru -Verb RunAs -Wait Get-Content $logFile -Tail 500 diff --git a/pkg/file/scripts/remove_exe.ps1 b/pkg/file/scripts/remove_exe.ps1 index eae826c960..abb41892c0 100644 --- a/pkg/file/scripts/remove_exe.ps1 +++ b/pkg/file/scripts/remove_exe.ps1 @@ -1,4 +1,4 @@ -$exeFilePath = "$INSTALLER_PATH" +$exeFilePath = "${env:INSTALLER_PATH}" # extract the name of the executable to use as the sub-directory name $exeName = [System.IO.Path]::GetFileName($exeFilePath) diff --git a/pkg/file/scripts/remove_msi.ps1 b/pkg/file/scripts/remove_msi.ps1 index 23172920ac..dc659508dc 100644 --- a/pkg/file/scripts/remove_msi.ps1 +++ b/pkg/file/scripts/remove_msi.ps1 @@ -1,7 +1,7 @@ $logFile = "${env:TEMP}/fleet-remove-software.log" $removeProcess = Start-Process msiexec.exe ` - -ArgumentList "/quiet /norestart /lv ${logFile} /x `"$INSTALLER_PATH`"" ` + -ArgumentList "/quiet /norestart /lv ${logFile} /x `"${env:INSTALLER_PATH}`"" ` -PassThru -Verb RunAs -Wait Get-Content $logFile -Tail 500 diff --git a/pkg/file/testdata/scripts/install_exe.ps1.golden b/pkg/file/testdata/scripts/install_exe.ps1.golden index e205c4ed11..f9d762b318 100644 --- a/pkg/file/testdata/scripts/install_exe.ps1.golden +++ b/pkg/file/testdata/scripts/install_exe.ps1.golden @@ -1,4 +1,4 @@ -$exeFilePath = "$INSTALLER_PATH" +$exeFilePath = "${env:INSTALLER_PATH}" # extract the name of the executable to use as the sub-directory name $exeName = [System.IO.Path]::GetFileName($exeFilePath) diff --git a/pkg/file/testdata/scripts/install_msi.ps1.golden b/pkg/file/testdata/scripts/install_msi.ps1.golden index 0f7595481d..838c431c1d 100644 --- a/pkg/file/testdata/scripts/install_msi.ps1.golden +++ b/pkg/file/testdata/scripts/install_msi.ps1.golden @@ -1,7 +1,7 @@ $logFile = "${env:TEMP}/fleet-install-software.log" $installProcess = Start-Process msiexec.exe ` - -ArgumentList "/quiet /norestart /lv ${logFile} /i `"$INSTALLER_PATH`"" ` + -ArgumentList "/quiet /norestart /lv ${logFile} /i `"${env:INSTALLER_PATH}`"" ` -PassThru -Verb RunAs -Wait Get-Content $logFile -Tail 500 diff --git a/pkg/file/testdata/scripts/remove_exe.ps1.golden b/pkg/file/testdata/scripts/remove_exe.ps1.golden index eae826c960..abb41892c0 100644 --- a/pkg/file/testdata/scripts/remove_exe.ps1.golden +++ b/pkg/file/testdata/scripts/remove_exe.ps1.golden @@ -1,4 +1,4 @@ -$exeFilePath = "$INSTALLER_PATH" +$exeFilePath = "${env:INSTALLER_PATH}" # extract the name of the executable to use as the sub-directory name $exeName = [System.IO.Path]::GetFileName($exeFilePath) diff --git a/pkg/file/testdata/scripts/remove_msi.ps1.golden b/pkg/file/testdata/scripts/remove_msi.ps1.golden index 23172920ac..dc659508dc 100644 --- a/pkg/file/testdata/scripts/remove_msi.ps1.golden +++ b/pkg/file/testdata/scripts/remove_msi.ps1.golden @@ -1,7 +1,7 @@ $logFile = "${env:TEMP}/fleet-remove-software.log" $removeProcess = Start-Process msiexec.exe ` - -ArgumentList "/quiet /norestart /lv ${logFile} /x `"$INSTALLER_PATH`"" ` + -ArgumentList "/quiet /norestart /lv ${logFile} /x `"${env:INSTALLER_PATH}`"" ` -PassThru -Verb RunAs -Wait Get-Content $logFile -Tail 500 diff --git a/server/datastore/mysql/scripts.go b/server/datastore/mysql/scripts.go index 97305ff1c8..4ecf682f15 100644 --- a/server/datastore/mysql/scripts.go +++ b/server/datastore/mysql/scripts.go @@ -66,6 +66,20 @@ func newHostScriptExecutionRequest(ctx context.Context, request *fleet.HostScrip return &script, nil } +func truncateScriptResult(output string) string { + const maxOutputRuneLen = 10000 + if len(output) > utf8.UTFMax*maxOutputRuneLen { + // truncate the bytes as we know the output is too long, no point + // converting more bytes than needed to runes. + output = output[len(output)-(utf8.UTFMax*maxOutputRuneLen):] + } + if utf8.RuneCountInString(output) > maxOutputRuneLen { + outputRunes := []rune(output) + output = string(outputRunes[len(outputRunes)-maxOutputRuneLen:]) + } + return output +} + func (ds *Datastore) SetHostScriptExecutionResult(ctx context.Context, result *fleet.HostScriptResultPayload) (*fleet.HostScriptResult, error) { const resultExistsStmt = ` SELECT @@ -101,17 +115,7 @@ func (ds *Datastore) SetHostScriptExecutionResult(ctx context.Context, result *f host_id = ? ` - const maxOutputRuneLen = 10000 - output := result.Output - if len(output) > utf8.UTFMax*maxOutputRuneLen { - // truncate the bytes as we know the output is too long, no point - // converting more bytes than needed to runes. - output = output[len(output)-(utf8.UTFMax*maxOutputRuneLen):] - } - if utf8.RuneCountInString(output) > maxOutputRuneLen { - outputRunes := []rune(output) - output = string(outputRunes[len(outputRunes)-maxOutputRuneLen:]) - } + output := truncateScriptResult(result.Output) var hsr *fleet.HostScriptResult err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 95168e4a9e..4fed1a2b91 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -14,6 +14,7 @@ import ( _ "github.com/doug-martin/goqu/v9/dialect/mysql" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" "github.com/go-kit/kit/log/level" "github.com/jmoiron/sqlx" ) @@ -2049,12 +2050,20 @@ func (ds *Datastore) SetHostSoftwareInstallResult(ctx context.Context, result *f execution_id = ? AND host_id = ? ` + + truncateOutput := func(output *string) *string { + if output != nil { + output = ptr.String(truncateScriptResult(*output)) + } + return output + } + res, err := ds.writer(ctx).ExecContext(ctx, stmt, - result.PreInstallConditionOutput, + truncateOutput(result.PreInstallConditionOutput), result.InstallScriptExitCode, - result.InstallScriptOutput, + truncateOutput(result.InstallScriptOutput), result.PostInstallScriptExitCode, - result.PostInstallScriptOutput, + truncateOutput(result.PostInstallScriptOutput), result.InstallUUID, result.HostID, ) From 8612e1aa59438c982da2c3e37792a7fa139e91a1 Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Wed, 15 May 2024 20:03:08 -0300 Subject: [PATCH 55/56] rearrange migrations --- ...bles.go => 20240515200020_AddSoftwareInstallerTables.go} | 6 +++--- server/datastore/mysql/schema.sql | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename server/datastore/mysql/migrations/tables/{20240424124712_AddSoftwareInstallerTables.go => 20240515200020_AddSoftwareInstallerTables.go} (96%) diff --git a/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go b/server/datastore/mysql/migrations/tables/20240515200020_AddSoftwareInstallerTables.go similarity index 96% rename from server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go rename to server/datastore/mysql/migrations/tables/20240515200020_AddSoftwareInstallerTables.go index 326487b371..a4d316ab20 100644 --- a/server/datastore/mysql/migrations/tables/20240424124712_AddSoftwareInstallerTables.go +++ b/server/datastore/mysql/migrations/tables/20240515200020_AddSoftwareInstallerTables.go @@ -6,10 +6,10 @@ import ( ) func init() { - MigrationClient.AddMigration(Up_20240424124712, Down_20240424124712) + MigrationClient.AddMigration(Up_20240515200020, Down_20240515200020) } -func Up_20240424124712(tx *sql.Tx) error { +func Up_20240515200020(tx *sql.Tx) error { _, err := tx.Exec(` CREATE TABLE IF NOT EXISTS software_installers ( id int(10) unsigned NOT NULL AUTO_INCREMENT, @@ -142,6 +142,6 @@ CREATE TABLE IF NOT EXISTS host_software_installs ( return nil } -func Down_20240424124712(tx *sql.Tx) error { +func Down_20240515200020(tx *sql.Tx) error { return nil } diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 2e6059b14c..15d5fe2e2f 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -912,7 +912,7 @@ CREATE TABLE `migration_status_tables` ( UNIQUE KEY `id` (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=266 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240424124712,1,'2020-01-01 01:01:01'),(265,20240430111727,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `mobile_device_management_solutions` ( From 8cc3d7dd4f9eb6cf79dca7de5ae8868d8d7f2fbe Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Thu, 16 May 2024 17:09:41 -0500 Subject: [PATCH 56/56] Update list host software to omit software installers for other platforms (#19088) --- ee/server/service/software_installers.go | 6 + ...240515200020_AddSoftwareInstallerTables.go | 7 +- server/datastore/mysql/schema.sql | 2 + server/datastore/mysql/software.go | 44 ++-- server/datastore/mysql/software_installers.go | 13 +- server/datastore/mysql/software_test.go | 248 ++++++++++++------ server/fleet/datastore.go | 2 +- server/fleet/software_installer.go | 17 ++ server/mock/datastore_mock.go | 6 +- server/service/hosts.go | 13 +- server/service/hosts_test.go | 2 +- server/service/integration_enterprise_test.go | 27 +- 12 files changed, 266 insertions(+), 121 deletions(-) diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index 2f8f8271af..bb8e51234c 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -344,6 +344,12 @@ func (svc *Service) addMetadataToSoftwarePayload(ctx context.Context, payload *f } payload.Source = source + platform, err := fleet.SofwareInstallerPlatformFromExtension(ext) + if err != nil { + return "", ctxerr.Wrap(ctx, err, "determining platform from extension") + } + payload.Platform = platform + return ext, nil } diff --git a/server/datastore/mysql/migrations/tables/20240515200020_AddSoftwareInstallerTables.go b/server/datastore/mysql/migrations/tables/20240515200020_AddSoftwareInstallerTables.go index a4d316ab20..5ef242c3f8 100644 --- a/server/datastore/mysql/migrations/tables/20240515200020_AddSoftwareInstallerTables.go +++ b/server/datastore/mysql/migrations/tables/20240515200020_AddSoftwareInstallerTables.go @@ -30,6 +30,9 @@ CREATE TABLE IF NOT EXISTS software_installers ( -- Version extracted from the uploaded installer version varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + -- Platform extracted from the uploaded installer + platform varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + -- Raw osquery SQL statment to be run as a pre-install condition pre_install_query text COLLATE utf8mb4_unicode_ci DEFAULT NULL, @@ -71,7 +74,9 @@ CREATE TABLE IF NOT EXISTS software_installers ( ON DELETE CASCADE ON UPDATE CASCADE, - UNIQUE KEY idx_software_installers_team_id_title_id (global_or_team_id, title_id) + UNIQUE KEY idx_software_installers_team_id_title_id (global_or_team_id, title_id), + + INDEX idx_software_installers_platform_title_id (platform, title_id) ) `) diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 15d5fe2e2f..791888aa10 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -1507,6 +1507,7 @@ CREATE TABLE `software_installers` ( `title_id` int(10) unsigned DEFAULT NULL, `filename` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, `version` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + `platform` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, `pre_install_query` text COLLATE utf8mb4_unicode_ci, `install_script_content_id` int(10) unsigned NOT NULL, `post_install_script_content_id` int(10) unsigned DEFAULT NULL, @@ -1518,6 +1519,7 @@ CREATE TABLE `software_installers` ( KEY `fk_software_installers_install_script_content_id` (`install_script_content_id`), KEY `fk_software_installers_post_install_script_content_id` (`post_install_script_content_id`), KEY `fk_software_installers_team_id` (`team_id`), + KEY `idx_software_installers_platform_title_id` (`platform`,`title_id`), CONSTRAINT `fk_software_installers_install_script_content_id` FOREIGN KEY (`install_script_content_id`) REFERENCES `script_contents` (`id`) ON UPDATE CASCADE, CONSTRAINT `fk_software_installers_post_install_script_content_id` FOREIGN KEY (`post_install_script_content_id`) REFERENCES `script_contents` (`id`) ON UPDATE CASCADE, CONSTRAINT `fk_software_installers_team_id` FOREIGN KEY (`team_id`) REFERENCES `teams` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 9f80eee1e3..eb018cb2fe 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -374,7 +374,6 @@ func (ds *Datastore) applyChangesForNewSoftwareDB( err = ds.withRetryTxx( ctx, func(tx sqlx.ExtContext) error { - deleted, err := deleteUninstalledHostSoftwareDB(ctx, tx, hostID, current, incoming) if err != nil { return err @@ -1848,7 +1847,7 @@ func softwareInstallerHostStatusNamedQuery(tblAlias, colAlias string) string { END %[2]s `, tblAlias, colAlias) } -func (ds *Datastore) ListHostSoftware(ctx context.Context, hostID uint, includeAvailableForInstall bool, opts fleet.ListOptions) ([]*fleet.HostSoftwareWithInstaller, *fleet.PaginationMetadata, error) { +func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, includeAvailableForInstall bool, opts fleet.ListOptions) ([]*fleet.HostSoftwareWithInstaller, *fleet.PaginationMetadata, error) { // `status` computed column assumes that all results (pre, install and post) // are stored at once, so that if there is an exit code for the install // script and none for the post-install, it is because there is no @@ -1905,7 +1904,8 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, hostID uint, includeA FROM software_titles st INNER JOIN - software_installers si ON st.id = si.title_id + -- filter out software that is not available for install on the host's platform + software_installers si ON st.id = si.title_id AND si.platform IN(%s) WHERE -- software is not installed on host, but is available in host's team NOT EXISTS ( @@ -1915,7 +1915,7 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, hostID uint, includeA INNER JOIN software s ON hs.software_id = s.id WHERE - hs.host_id = :host_id AND + hs.host_id = ? AND s.title_id = st.id ) AND -- sofware install has not been attempted on host @@ -1924,10 +1924,10 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, hostID uint, includeA FROM host_software_installs hsi WHERE - hsi.host_id = :host_id AND + hsi.host_id = ? AND hsi.software_installer_id = si.id ) AND - si.global_or_team_id = (SELECT COALESCE(h.team_id, 0) FROM hosts h WHERE h.id = :host_id) + si.global_or_team_id = (SELECT COALESCE(h.team_id, 0) FROM hosts h WHERE h.id = ?) ` const selectColNames = ` @@ -1941,16 +1941,10 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, hostID uint, includeA status ` - stmt := stmtInstalled - if includeAvailableForInstall { - stmt += ` UNION ` + stmtAvailable - } - - stmt = selectColNames + ` FROM ( ` + stmt + ` ) AS tbl ` - // must resolve the named bindings here, before adding the searchLike which + // must resolve the named bindings here, before adding the stmtAvailable and searchLike which // uses standard placeholders. - stmt, args, err := sqlx.Named(stmt, map[string]any{ - "host_id": hostID, + stmt, args, err := sqlx.Named(stmtInstalled, map[string]any{ + "host_id": host.ID, "software_status_failed": fleet.SoftwareInstallerFailed, "software_status_pending": fleet.SoftwareInstallerPending, "software_status_installed": fleet.SoftwareInstallerInstalled, @@ -1959,6 +1953,22 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, hostID uint, includeA return nil, nil, ctxerr.Wrap(ctx, err, "build named query for list host software") } + if includeAvailableForInstall { + platformArgs := []string{host.Platform} + if fleet.IsLinux(host.Platform) { + platformArgs = fleet.HostLinuxOSs + } + placeholders := "" + for _, p := range platformArgs { + placeholders += "?," + args = append(args, p) + } + stmt += ` UNION ` + fmt.Sprintf(stmtAvailable, strings.TrimSuffix(placeholders, ",")) + args = append(args, host.ID, host.ID, host.ID) + } + + stmt = selectColNames + ` FROM ( ` + stmt + ` ) AS tbl ` + if opts.MatchQuery != "" { stmt += " WHERE TRUE " // searchLike adds a "AND " stmt, args = searchLike(stmt, args, opts.MatchQuery, "name") @@ -2021,7 +2031,7 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, hostID uint, includeA st.id IN (?) ` var installedVersions []*fleet.HostSoftwareInstalledVersion - stmt, args, err := sqlx.In(versionStmt, hostID, titleIDs) + stmt, args, err := sqlx.In(versionStmt, host.ID, titleIDs) if err != nil { return nil, nil, ctxerr.Wrap(ctx, err, "building query args to list versions") } @@ -2088,7 +2098,7 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, hostID uint, includeA InstalledPath string `db:"installed_path"` } var installedPaths []installedPath - stmt, args, err = sqlx.In(pathsStmt, hostID, softwareIDs) + stmt, args, err = sqlx.In(pathsStmt, host.ID, softwareIDs) if err != nil { return nil, nil, ctxerr.Wrap(ctx, err, "building query args to list installed paths") } diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index a19a4b35bc..8eef7a7374 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -103,8 +103,9 @@ INSERT INTO software_installers ( version, install_script_content_id, pre_install_query, - post_install_script_content_id -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + post_install_script_content_id, + platform +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` args := []interface{}{ payload.TeamID, @@ -116,6 +117,7 @@ INSERT INTO software_installers ( installScriptID, payload.PreInstallQuery, postInstallScriptID, + payload.Platform, } res, err := ds.writer(ctx).ExecContext(ctx, stmt, args...) @@ -460,9 +462,10 @@ INSERT INTO software_installers ( install_script_content_id, pre_install_query, post_install_script_content_id, + platform, title_id ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, (SELECT id FROM software_titles WHERE name = ? AND source = ? AND browser = '') ) ON DUPLICATE KEY UPDATE @@ -471,7 +474,8 @@ ON DUPLICATE KEY UPDATE storage_id = VALUES(storage_id), filename = VALUES(filename), version = VALUES(version), - pre_install_query = VALUES(pre_install_query) + pre_install_query = VALUES(pre_install_query), + platform = VALUES(platform) ` // use a team id of 0 if no-team @@ -541,6 +545,7 @@ ON DUPLICATE KEY UPDATE installScriptID, installer.PreInstallQuery, postInstallScriptID, + installer.Platform, installer.Title, installer.Source, } diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index e0b2d9499e..aafda020fa 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -2997,7 +2997,7 @@ func testVerifySoftwareChecksum(t *testing.T, ds *Datastore) { func testListHostSoftware(t *testing.T, ds *Datastore) { ctx := context.Background() - host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) + host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now(), test.WithPlatform("linux")) otherHost := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now()) opts := fleet.ListOptions{PerPage: 10, IncludeMetadata: true, OrderKey: "name", TestSecondaryOrderKey: "source"} @@ -3006,13 +3006,13 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { } // no software yet - sw, meta, err := ds.ListHostSoftware(ctx, host.ID, false, opts) + sw, meta, err := ds.ListHostSoftware(ctx, host, false, opts) require.NoError(t, err) require.Empty(t, sw) require.Equal(t, &fleet.PaginationMetadata{}, meta) // works with available software too - sw, meta, err = ds.ListHostSoftware(ctx, host.ID, true, opts) + sw, meta, err = ds.ListHostSoftware(ctx, host, true, opts) require.NoError(t, err) require.Empty(t, sw) require.Equal(t, &fleet.PaginationMetadata{}, meta) @@ -3026,9 +3026,37 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { {Name: "c", Version: "0.0.5", Source: "deb_packages"}, {Name: "d", Version: "0.0.6", Source: "deb_packages"}, } + byNSV := map[string]fleet.Software{} + for _, s := range software { + byNSV[s.Name+s.Source+s.Version] = s + } + mutationResults, err := ds.UpdateHostSoftware(ctx, host.ID, software) require.NoError(t, err) + require.Len(t, mutationResults.Inserted, len(software)) + for _, m := range mutationResults.Inserted { + s, ok := byNSV[m.Name+m.Source+m.Version] + require.True(t, ok) + require.Equal(t, m.Name, s.Name, "name") + require.Equal(t, m.Version, s.Version, "version") + require.Equal(t, m.Source, s.Source, "source") + require.Zero(t, s.ID) // not set in the map yet + require.NotZero(t, m.ID) + s.ID = m.ID + byNSV[s.Name+s.Source+s.Version] = s + + } + require.NoError(t, ds.LoadHostSoftware(ctx, host, false)) + require.Equal(t, len(host.Software), len(software)) + for _, hs := range host.Software { + s, ok := byNSV[hs.Name+hs.Source+hs.Version] + require.True(t, ok) + require.Equal(t, hs.Name, s.Name, "name") + require.Equal(t, hs.Version, s.Version, "version") + require.Equal(t, hs.Source, s.Source, "source") + require.Equal(t, hs.ID, s.ID) + } // add other software to the other host, won't be returned otherSoftware := []fleet.Software{ @@ -3038,12 +3066,20 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { _, err = ds.UpdateHostSoftware(ctx, otherHost.ID, otherSoftware) require.NoError(t, err) + // shorthand keys for expected software + a1 := software[0].Name + software[0].Source + software[0].Version + a2 := software[1].Name + software[1].Source + software[1].Version + b := software[2].Name + software[2].Source + software[2].Version + c1 := software[3].Name + software[3].Source + software[3].Version + c2 := software[4].Name + software[4].Source + software[4].Version + d := software[5].Name + software[5].Source + software[5].Version + // add some vulnerabilities and installed paths vulns := []fleet.SoftwareVulnerability{ - {SoftwareID: host.Software[0].ID, CVE: "CVE-a-0001"}, - {SoftwareID: host.Software[0].ID, CVE: "CVE-a-0002"}, - {SoftwareID: host.Software[0].ID, CVE: "CVE-a-0003"}, - {SoftwareID: host.Software[2].ID, CVE: "CVE-b-0001"}, + {SoftwareID: byNSV[a1].ID, CVE: "CVE-a-0001"}, + {SoftwareID: byNSV[a1].ID, CVE: "CVE-a-0002"}, + {SoftwareID: byNSV[a1].ID, CVE: "CVE-a-0003"}, + {SoftwareID: byNSV[b].ID, CVE: "CVE-b-0001"}, } for _, v := range vulns { _, err = ds.InsertSoftwareVulnerability(ctx, v, fleet.NVDSource) @@ -3064,48 +3100,65 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { err = ds.ReconcileSoftwareTitles(ctx) require.NoError(t, err) - expected := map[string]*fleet.HostSoftwareWithInstaller{ - "a1": {Name: software[0].Name, Source: software[0].Source, InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ - {Version: software[0].Version, Vulnerabilities: []string{vulns[0].CVE, vulns[1].CVE, vulns[2].CVE}, InstalledPaths: []string{installPaths[0]}}, + expected := map[string]fleet.HostSoftwareWithInstaller{ + byNSV[a1].Name + byNSV[a1].Source: {Name: byNSV[a1].Name, Source: byNSV[a1].Source, InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ + {Version: byNSV[a1].Version, Vulnerabilities: []string{vulns[0].CVE, vulns[1].CVE, vulns[2].CVE}, InstalledPaths: []string{installPaths[0]}}, }}, - "a2": {Name: software[1].Name, Source: software[1].Source, InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ - {Version: software[1].Version, InstalledPaths: []string{installPaths[1]}}, + // a1 and a2 are different software titles because they have different sources + byNSV[a2].Name + byNSV[a2].Source: {Name: byNSV[a2].Name, Source: byNSV[a2].Source, InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ + {Version: byNSV[a2].Version, InstalledPaths: []string{installPaths[1]}}, }}, - "b": {Name: software[2].Name, Source: software[2].Source, InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ - {Version: software[2].Version, Vulnerabilities: []string{vulns[3].CVE}, InstalledPaths: []string{installPaths[2]}}, + byNSV[b].Name + byNSV[b].Source: {Name: byNSV[b].Name, Source: byNSV[b].Source, InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ + {Version: byNSV[b].Version, Vulnerabilities: []string{vulns[3].CVE}, InstalledPaths: []string{installPaths[2]}}, }}, - "c": {Name: software[3].Name, Source: software[3].Source, InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ - {Version: software[3].Version, InstalledPaths: []string{installPaths[3]}}, - {Version: software[4].Version, InstalledPaths: []string{installPaths[4]}}, + // c1 and c2 are the same software title because they have the same name and source + byNSV[c1].Name + byNSV[c1].Source: {Name: byNSV[c1].Name, Source: byNSV[c1].Source, InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ + {Version: byNSV[c1].Version, InstalledPaths: []string{installPaths[3]}}, + {Version: byNSV[c2].Version, InstalledPaths: []string{installPaths[4]}}, }}, - "d": {Name: software[5].Name, Source: software[5].Source, InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ - {Version: software[5].Version, InstalledPaths: []string{installPaths[5]}}, + byNSV[d].Name + byNSV[d].Source: {Name: byNSV[d].Name, Source: byNSV[d].Source, InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ + {Version: byNSV[d].Version, InstalledPaths: []string{installPaths[5]}}, }}, } - compareResults := func(expected, got []*fleet.HostSoftwareWithInstaller) { - require.Len(t, got, len(expected)) - // clear ids and timestamps for comparison + compareResults := func(expected map[string]fleet.HostSoftwareWithInstaller, got []*fleet.HostSoftwareWithInstaller, expectAsc bool, expectOmitted ...string) { + require.Len(t, got, len(expected)-len(expectOmitted)) + prev := "" for _, g := range got { - g.ID = 0 - if g.LastInstall != nil { - g.LastInstall.InstalledAt = time.Time{} + e, ok := expected[g.Name+g.Source] + require.True(t, ok) + require.Equal(t, e.Name, g.Name) + require.Equal(t, e.Source, g.Source) + require.Len(t, g.InstalledVersions, len(e.InstalledVersions)) + if len(e.InstalledVersions) > 0 { + byVers := make(map[string]fleet.HostSoftwareInstalledVersion, len(e.InstalledVersions)) + for _, v := range e.InstalledVersions { + byVers[v.Version] = *v + } + for _, v := range g.InstalledVersions { + ev, ok := byVers[v.Version] + require.True(t, ok) + require.Equal(t, ev.Version, v.Version) + require.ElementsMatch(t, ev.InstalledPaths, v.InstalledPaths) + require.ElementsMatch(t, ev.Vulnerabilities, v.Vulnerabilities) + } } - for _, v := range g.InstalledVersions { - v.SoftwareID = 0 - v.SoftwareTitleID = 0 + if prev != "" { + if expectAsc { + require.Greater(t, g.Name+g.Source, prev) + } else { + require.Less(t, g.Name+g.Source, prev) + } } + prev = g.Name + g.Source } - require.Equal(t, expected, got) } // it now returns the software with vulnerabilities and installed paths - sw, meta, err = ds.ListHostSoftware(ctx, host.ID, false, opts) + sw, meta, err = ds.ListHostSoftware(ctx, host, false, opts) require.NoError(t, err) require.Equal(t, &fleet.PaginationMetadata{TotalResults: 5}, meta) - compareResults([]*fleet.HostSoftwareWithInstaller{ - expected["a1"], expected["a2"], expected["b"], expected["c"], expected["d"], - }, sw) + compareResults(expected, sw, true) // create some Fleet installers and map them to a software title, // including one for a team @@ -3159,10 +3212,10 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { } res, err := q.ExecContext(ctx, ` INSERT INTO software_installers - (team_id, global_or_team_id, title_id, filename, version, install_script_content_id, storage_id) + (team_id, global_or_team_id, title_id, filename, version, install_script_content_id, storage_id, platform) VALUES - (?, ?, ?, ?, ?, ?, unhex(?))`, - teamID, globalOrTeamID, titleID, fmt.Sprintf("installer-%d.pkg", i), fmt.Sprintf("v%d.0.0", i), scriptContentID, hex.EncodeToString([]byte("test"))) + (?, ?, ?, ?, ?, ?, unhex(?), ?)`, + teamID, globalOrTeamID, titleID, fmt.Sprintf("installer-%d.pkg", i), fmt.Sprintf("v%d.0.0", i), scriptContentID, hex.EncodeToString([]byte("test")), "linux") if err != nil { return err } @@ -3216,74 +3269,91 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { // swi5 is for another team _ = swi5Tm + // add another installer for a different platform, should be always omitted + res, err = q.ExecContext(ctx, `INSERT INTO software_titles (name, source) VALUES ('windows-title', 'programs')`) + if err != nil { + return err + } + lid, _ := res.LastInsertId() + _, err = q.ExecContext(ctx, ` + INSERT INTO software_installers + (team_id, global_or_team_id, title_id, filename, version, install_script_content_id, storage_id, platform) + VALUES + (?, ?, ?, ?, ?, ?, unhex(?), ?)`, + nil, 0, lid, "windows-installer-6.msi", "v6.0.0", scriptContentID, hex.EncodeToString([]byte("test")), "windows") + if err != nil { + return err + } + return nil }) // swi1Pending uses software title id of "b" - expected["b"] = &fleet.HostSoftwareWithInstaller{ + expected[byNSV[b].Name+byNSV[b].Source] = fleet.HostSoftwareWithInstaller{ Name: "b", Source: "apps", Status: expectStatus(fleet.SoftwareInstallerPending), LastInstall: &fleet.HostSoftwareInstall{InstallUUID: "uuid1"}, PackageAvailableForInstall: ptr.String("installer-0.pkg"), InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ - {Version: software[2].Version, Vulnerabilities: []string{vulns[3].CVE}, InstalledPaths: []string{installPaths[2]}}, + {Version: byNSV[b].Version, Vulnerabilities: []string{vulns[3].CVE}, InstalledPaths: []string{installPaths[2]}}, }, } - expected["i0"] = &fleet.HostSoftwareWithInstaller{ + i0 := fleet.HostSoftwareWithInstaller{ Name: "i0", Source: "apps", Status: expectStatus(fleet.SoftwareInstallerInstalled), LastInstall: &fleet.HostSoftwareInstall{InstallUUID: "uuid2"}, PackageAvailableForInstall: ptr.String("installer-1.pkg"), } - expected["i1"] = &fleet.HostSoftwareWithInstaller{ + expected[i0.Name+i0.Source] = i0 + + i1 := fleet.HostSoftwareWithInstaller{ Name: "i1", Source: "apps", Status: expectStatus(fleet.SoftwareInstallerFailed), LastInstall: &fleet.HostSoftwareInstall{InstallUUID: "uuid3"}, PackageAvailableForInstall: ptr.String("installer-2.pkg"), } - expected["i2"] = &fleet.HostSoftwareWithInstaller{ + expected[i1.Name+i1.Source] = i1 + + // request without available software + sw, meta, err = ds.ListHostSoftware(ctx, host, false, opts) + require.NoError(t, err) + require.Equal(t, &fleet.PaginationMetadata{TotalResults: 7}, meta) + compareResults(expected, sw, true) + + // request with available software + i2 := fleet.HostSoftwareWithInstaller{ Name: "i2", Source: "apps", Status: nil, LastInstall: nil, PackageAvailableForInstall: ptr.String("installer-3.pkg"), } - expected["i3"] = &fleet.HostSoftwareWithInstaller{ + expected[i2.Name+i2.Source] = i2 + + i3 := fleet.HostSoftwareWithInstaller{ Name: "i3", Source: "apps", Status: nil, LastInstall: nil, PackageAvailableForInstall: ptr.String("installer-4.pkg"), } + expected[i3.Name+i3.Source] = i3 - // request without available software - sw, meta, err = ds.ListHostSoftware(ctx, host.ID, false, opts) - require.NoError(t, err) - require.Equal(t, &fleet.PaginationMetadata{TotalResults: 7}, meta) - compareResults([]*fleet.HostSoftwareWithInstaller{ - expected["a1"], expected["a2"], expected["b"], expected["c"], expected["d"], expected["i0"], expected["i1"], - }, sw) - - // request with available software - sw, meta, err = ds.ListHostSoftware(ctx, host.ID, true, opts) + sw, meta, err = ds.ListHostSoftware(ctx, host, true, opts) require.NoError(t, err) require.Equal(t, &fleet.PaginationMetadata{TotalResults: 8}, meta) - compareResults([]*fleet.HostSoftwareWithInstaller{ - expected["a1"], expected["a2"], expected["b"], expected["c"], expected["d"], expected["i0"], expected["i1"], expected["i2"], - }, sw) + compareResults(expected, sw, true, i3.Name+i3.Source) // request in descending order opts.OrderDirection = fleet.OrderDescending opts.TestSecondaryOrderDirection = fleet.OrderDescending - sw, meta, err = ds.ListHostSoftware(ctx, host.ID, false, opts) + sw, meta, err = ds.ListHostSoftware(ctx, host, false, opts) require.NoError(t, err) require.Equal(t, &fleet.PaginationMetadata{TotalResults: 7}, meta) - compareResults([]*fleet.HostSoftwareWithInstaller{ - expected["i1"], expected["i0"], expected["d"], expected["c"], expected["b"], expected["a2"], expected["a1"], - }, sw) + compareResults(expected, sw, false, i2.Name+i2.Source, i3.Name+i3.Source) opts.OrderDirection = fleet.OrderAscending opts.TestSecondaryOrderDirection = fleet.OrderAscending @@ -3308,17 +3378,17 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { return nil }) - expected["b"] = &fleet.HostSoftwareWithInstaller{ + expected[byNSV[b].Name+byNSV[b].Source] = fleet.HostSoftwareWithInstaller{ Name: "b", Source: "apps", Status: expectStatus(fleet.SoftwareInstallerFailed), LastInstall: &fleet.HostSoftwareInstall{InstallUUID: "uuid1"}, PackageAvailableForInstall: ptr.String("installer-0.pkg"), InstalledVersions: []*fleet.HostSoftwareInstalledVersion{ - {Version: software[2].Version, Vulnerabilities: []string{vulns[3].CVE}, InstalledPaths: []string{installPaths[2]}}, + {Version: byNSV[b].Version, Vulnerabilities: []string{vulns[3].CVE}, InstalledPaths: []string{installPaths[2]}}, }, } - expected["i1"] = &fleet.HostSoftwareWithInstaller{ + expected[i1.Name+i1.Source] = fleet.HostSoftwareWithInstaller{ Name: "i1", Source: "apps", Status: expectStatus(fleet.SoftwareInstallerPending), @@ -3327,52 +3397,56 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { } // request without available software - sw, meta, err = ds.ListHostSoftware(ctx, host.ID, false, opts) + sw, meta, err = ds.ListHostSoftware(ctx, host, false, opts) require.NoError(t, err) require.Equal(t, &fleet.PaginationMetadata{TotalResults: 7}, meta) - compareResults([]*fleet.HostSoftwareWithInstaller{ - expected["a1"], expected["a2"], expected["b"], expected["c"], expected["d"], expected["i0"], expected["i1"], - }, sw) + compareResults(expected, sw, true, i2.Name+i2.Source, i3.Name+i3.Source) - // request with available software - sw, meta, err = ds.ListHostSoftware(ctx, host.ID, true, opts) + // request with available software) + sw, meta, err = ds.ListHostSoftware(ctx, host, true, opts) require.NoError(t, err) require.Equal(t, &fleet.PaginationMetadata{TotalResults: 8}, meta) - compareResults([]*fleet.HostSoftwareWithInstaller{ - expected["a1"], expected["a2"], expected["b"], expected["c"], expected["d"], expected["i0"], expected["i1"], expected["i2"], - }, sw) + compareResults(expected, sw, true, i3.Name+i3.Source) // create a new host in the team, with no software - tmHost := test.NewHost(t, ds, "host3", "", "host3key", "host3uuid", time.Now()) + tmHost := test.NewHost(t, ds, "host3", "", "host3key", "host3uuid", time.Now(), test.WithPlatform("linux")) err = ds.AddHostsToTeam(ctx, &tm.ID, []uint{tmHost.ID}) require.NoError(t, err) // no installed software for this host - sw, meta, err = ds.ListHostSoftware(ctx, tmHost.ID, false, opts) + sw, meta, err = ds.ListHostSoftware(ctx, tmHost, false, opts) require.NoError(t, err) require.Empty(t, sw) require.Equal(t, &fleet.PaginationMetadata{}, meta) // sees the available installer in its team - sw, meta, err = ds.ListHostSoftware(ctx, tmHost.ID, true, opts) + sw, meta, err = ds.ListHostSoftware(ctx, tmHost, true, opts) require.NoError(t, err) require.Equal(t, &fleet.PaginationMetadata{TotalResults: 1}, meta) - compareResults([]*fleet.HostSoftwareWithInstaller{expected["i3"]}, sw) + compareResults(map[string]fleet.HostSoftwareWithInstaller{ + i3.Name + i3.Source: expected[i3.Name+i3.Source], + }, sw, true) // test with a search query (searches on name), with and without available software opts.MatchQuery = "a" - sw, _, err = ds.ListHostSoftware(ctx, host.ID, false, opts) + sw, _, err = ds.ListHostSoftware(ctx, host, false, opts) require.NoError(t, err) - compareResults([]*fleet.HostSoftwareWithInstaller{expected["a1"], expected["a2"]}, sw) - sw, _, err = ds.ListHostSoftware(ctx, host.ID, true, opts) + compareResults(map[string]fleet.HostSoftwareWithInstaller{ + byNSV[a1].Name + byNSV[a1].Source: expected[byNSV[a1].Name+byNSV[a1].Source], + byNSV[a2].Name + byNSV[a2].Source: expected[byNSV[a2].Name+byNSV[a2].Source], + }, sw, true) + sw, _, err = ds.ListHostSoftware(ctx, host, true, opts) require.NoError(t, err) - compareResults([]*fleet.HostSoftwareWithInstaller{expected["a1"], expected["a2"]}, sw) + compareResults(map[string]fleet.HostSoftwareWithInstaller{ + byNSV[a1].Name + byNSV[a1].Source: expected[byNSV[a1].Name+byNSV[a1].Source], + byNSV[a2].Name + byNSV[a2].Source: expected[byNSV[a2].Name+byNSV[a2].Source], + }, sw, true) opts.MatchQuery = "zz" - sw, _, err = ds.ListHostSoftware(ctx, host.ID, false, opts) + sw, _, err = ds.ListHostSoftware(ctx, host, false, opts) require.NoError(t, err) require.Empty(t, sw) - sw, _, err = ds.ListHostSoftware(ctx, host.ID, true, opts) + sw, _, err = ds.ListHostSoftware(ctx, host, true, opts) require.NoError(t, err) require.Empty(t, sw) @@ -3386,19 +3460,19 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { { opts: fleet.ListOptions{PerPage: 3}, withAvailable: false, - wantNames: []string{expected["a1"].Name, expected["a2"].Name, expected["b"].Name}, + wantNames: []string{byNSV[a1].Name, byNSV[a2].Name, byNSV[b].Name}, wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false, TotalResults: 7}, }, { opts: fleet.ListOptions{Page: 1, PerPage: 3}, withAvailable: false, - wantNames: []string{expected["c"].Name, expected["d"].Name, expected["i0"].Name}, + wantNames: []string{byNSV[c1].Name, byNSV[d].Name, i0.Name}, wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: true, TotalResults: 7}, }, { opts: fleet.ListOptions{Page: 2, PerPage: 3}, withAvailable: false, - wantNames: []string{expected["i1"].Name}, + wantNames: []string{i1.Name}, wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 7}, }, { @@ -3410,13 +3484,13 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { { opts: fleet.ListOptions{PerPage: 4}, withAvailable: true, - wantNames: []string{expected["a1"].Name, expected["a2"].Name, expected["b"].Name, expected["c"].Name}, + wantNames: []string{byNSV[a1].Name, byNSV[a2].Name, byNSV[b].Name, byNSV[c1].Name}, wantMeta: &fleet.PaginationMetadata{HasNextResults: true, HasPreviousResults: false, TotalResults: 8}, }, { opts: fleet.ListOptions{Page: 1, PerPage: 4}, withAvailable: true, - wantNames: []string{expected["d"].Name, expected["i0"].Name, expected["i1"].Name, expected["i2"].Name}, + wantNames: []string{byNSV[d].Name, i0.Name, i1.Name, i2.Name}, wantMeta: &fleet.PaginationMetadata{HasNextResults: false, HasPreviousResults: true, TotalResults: 8}, }, { @@ -3433,7 +3507,7 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { c.opts.OrderKey = "name" c.opts.TestSecondaryOrderKey = "source" - sw, meta, err := ds.ListHostSoftware(ctx, host.ID, c.withAvailable, c.opts) + sw, meta, err := ds.ListHostSoftware(ctx, host, c.withAvailable, c.opts) require.NoError(t, err) require.Equal(t, len(c.wantNames), len(sw)) diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index e5be418ff0..0c48de6370 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -551,7 +551,7 @@ type Datastore interface { InsertCVEMeta(ctx context.Context, cveMeta []CVEMeta) error ListCVEs(ctx context.Context, maxAge time.Duration) ([]CVEMeta, error) - ListHostSoftware(ctx context.Context, hostID uint, includeAvailableForInstall bool, opts ListOptions) ([]*HostSoftwareWithInstaller, *PaginationMetadata, error) + ListHostSoftware(ctx context.Context, host *Host, includeAvailableForInstall bool, opts ListOptions) ([]*HostSoftwareWithInstaller, *PaginationMetadata, error) // SetHostSoftwareInstallResult records the result of a software installation // attempt on the host. diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index 517a434a92..068e539fc6 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "strings" "time" "github.com/fleetdm/fleet/v4/server/ptr" @@ -250,6 +251,7 @@ type UploadSoftwareInstallerPayload struct { Title string Version string Source string + Platform string } // DownloadSoftwareInstallerPayload is the payload for downloading a software installer. @@ -260,6 +262,7 @@ type DownloadSoftwareInstallerPayload struct { } func SofwareInstallerSourceFromExtension(ext string) (string, error) { + ext = strings.TrimPrefix(ext, ".") switch ext { case "deb": return "deb_packages", nil @@ -272,6 +275,20 @@ func SofwareInstallerSourceFromExtension(ext string) (string, error) { } } +func SofwareInstallerPlatformFromExtension(ext string) (string, error) { + ext = strings.TrimPrefix(ext, ".") + switch ext { + case "deb": + return "linux", nil + case "exe", "msi": + return "windows", nil + case "pkg": + return "darwin", nil + default: + return "", fmt.Errorf("unsupported file type: %s", ext) + } +} + // HostSoftwareWithInstaller represents the list of software installed on a // host with installer information if a matching installer exists. This is the // payload returned by the "Get host's (device's) software" endpoints. diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 2327566745..d37ee164ad 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -403,7 +403,7 @@ type InsertCVEMetaFunc func(ctx context.Context, cveMeta []fleet.CVEMeta) error type ListCVEsFunc func(ctx context.Context, maxAge time.Duration) ([]fleet.CVEMeta, error) -type ListHostSoftwareFunc func(ctx context.Context, hostID uint, includeAvailableForInstall bool, opts fleet.ListOptions) ([]*fleet.HostSoftwareWithInstaller, *fleet.PaginationMetadata, error) +type ListHostSoftwareFunc func(ctx context.Context, host *fleet.Host, includeAvailableForInstall bool, opts fleet.ListOptions) ([]*fleet.HostSoftwareWithInstaller, *fleet.PaginationMetadata, error) type SetHostSoftwareInstallResultFunc func(ctx context.Context, result *fleet.HostSoftwareInstallResultPayload) error @@ -3702,11 +3702,11 @@ func (s *DataStore) ListCVEs(ctx context.Context, maxAge time.Duration) ([]fleet return s.ListCVEsFunc(ctx, maxAge) } -func (s *DataStore) ListHostSoftware(ctx context.Context, hostID uint, includeAvailableForInstall bool, opts fleet.ListOptions) ([]*fleet.HostSoftwareWithInstaller, *fleet.PaginationMetadata, error) { +func (s *DataStore) ListHostSoftware(ctx context.Context, host *fleet.Host, includeAvailableForInstall bool, opts fleet.ListOptions) ([]*fleet.HostSoftwareWithInstaller, *fleet.PaginationMetadata, error) { s.mu.Lock() s.ListHostSoftwareFuncInvoked = true s.mu.Unlock() - return s.ListHostSoftwareFunc(ctx, hostID, includeAvailableForInstall, opts) + return s.ListHostSoftwareFunc(ctx, host, includeAvailableForInstall, opts) } func (s *DataStore) SetHostSoftwareInstallResult(ctx context.Context, result *fleet.HostSoftwareInstallResultPayload) error { diff --git a/server/service/hosts.go b/server/service/hosts.go index d932bfb553..5d51946edf 100644 --- a/server/service/hosts.go +++ b/server/service/hosts.go @@ -17,6 +17,7 @@ import ( "github.com/fleetdm/fleet/v4/server/authz" authzctx "github.com/fleetdm/fleet/v4/server/contexts/authz" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + hostctx "github.com/fleetdm/fleet/v4/server/contexts/host" "github.com/fleetdm/fleet/v4/server/contexts/license" "github.com/fleetdm/fleet/v4/server/contexts/logging" "github.com/fleetdm/fleet/v4/server/contexts/viewer" @@ -2498,6 +2499,7 @@ func (svc *Service) ListHostSoftware(ctx context.Context, hostID uint, opts flee // that is not installed but for which there's an installer available for that host. var includeAvailableForInstall bool + var host *fleet.Host if !svc.authz.IsAuthenticatedWith(ctx, authzctx.AuthnDeviceToken) { includeAvailableForInstall = true @@ -2505,15 +2507,22 @@ func (svc *Service) ListHostSoftware(ctx context.Context, hostID uint, opts flee return nil, nil, err } - host, err := svc.ds.HostLite(ctx, hostID) + h, err := svc.ds.HostLite(ctx, hostID) if err != nil { return nil, nil, ctxerr.Wrap(ctx, err, "get host lite") } + host = h // Authorize again with team loaded now that we have team_id if err := svc.authz.Authorize(ctx, host, fleet.ActionRead); err != nil { return nil, nil, err } + } else { + h, ok := hostctx.FromContext(ctx) + if !ok { + return nil, nil, ctxerr.Wrap(ctx, fleet.NewAuthRequiredError("internal error: missing host from request context")) + } + host = h } // cursor-based pagination is not supported @@ -2523,7 +2532,7 @@ func (svc *Service) ListHostSoftware(ctx context.Context, hostID uint, opts flee // always include metadata opts.IncludeMetadata = true - software, meta, err := svc.ds.ListHostSoftware(ctx, hostID, includeAvailableForInstall, opts) + software, meta, err := svc.ds.ListHostSoftware(ctx, host, includeAvailableForInstall, opts) if !includeAvailableForInstall { // for the device page, we don't want to return the package name for _, s := range software { diff --git a/server/service/hosts_test.go b/server/service/hosts_test.go index 0f4bfcfcd2..49a824b335 100644 --- a/server/service/hosts_test.go +++ b/server/service/hosts_test.go @@ -618,7 +618,7 @@ func TestHostAuth(t *testing.T) { ds.GetHostLockWipeStatusFunc = func(ctx context.Context, host *fleet.Host) (*fleet.HostLockWipeStatus, error) { return &fleet.HostLockWipeStatus{}, nil } - ds.ListHostSoftwareFunc = func(ctx context.Context, hostID uint, includeAvailableForInstall bool, opts fleet.ListOptions) ([]*fleet.HostSoftwareWithInstaller, *fleet.PaginationMetadata, error) { + ds.ListHostSoftwareFunc = func(ctx context.Context, host *fleet.Host, includeAvailableForInstall bool, opts fleet.ListOptions) ([]*fleet.HostSoftwareWithInstaller, *fleet.PaginationMetadata, error) { return nil, nil, nil } diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 623fd9d1bc..e5a7aca8b7 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -8909,16 +8909,22 @@ func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerUploadDownloadAndD } checkSoftwareInstaller := func(t *testing.T, payload *fleet.UploadSoftwareInstallerPayload) (installerID uint, titleID uint) { + var tid uint + if payload.TeamID != nil { + tid = *payload.TeamID + } var id uint mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { - var tid uint - if payload.TeamID != nil { - tid = *payload.TeamID - } return sqlx.GetContext(context.Background(), q, &id, `SELECT id FROM software_installers WHERE global_or_team_id = ? AND filename = ?`, tid, payload.Filename) }) require.NotZero(t, id) + var platform string + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(context.Background(), q, &platform, `SELECT platform FROM software_installers WHERE id = ?`, id) + }) + require.Equal(t, payload.Platform, "linux") + meta, err := s.ds.GetSoftwareInstallerMetadataByID(context.Background(), id) require.NoError(t, err) @@ -8958,6 +8964,7 @@ func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerUploadDownloadAndD Version: "1:2.5.1", Source: "deb_packages", StorageID: "df06d9ce9e2090d9cb2e8cd1f4d7754a803dc452bf93e3204e3acd3b95508628", + Platform: "linux", } s.uploadSoftwareInstaller(payload, http.StatusOK, "") @@ -9002,6 +9009,7 @@ func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerUploadDownloadAndD Version: "1:2.5.1", Source: "deb_packages", StorageID: "df06d9ce9e2090d9cb2e8cd1f4d7754a803dc452bf93e3204e3acd3b95508628", + Platform: "linux", } s.uploadSoftwareInstaller(payload, http.StatusOK, "") @@ -9231,6 +9239,16 @@ func (s *integrationMDMTestSuite) TestBatchSetSoftwareInstallers() { require.Equal(t, 1, titlesResp.Count) require.Len(t, titlesResp.SoftwareTitles, 1) + // check that platform is set when the installer is created + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + var platform string + if err := sqlx.GetContext(context.Background(), q, &platform, `SELECT platform FROM software_installers WHERE title_id= ? AND team_id = ?`, titlesResp.SoftwareTitles[0].ID, tm.ID); err != nil { + return err + } + require.Equal(t, "linux", platform) + return nil + }) + // same payload doesn't modify anything s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusNoContent, "team_name", tm.Name) newTitlesResp := listSoftwareTitlesResponse{} @@ -9244,7 +9262,6 @@ func (s *integrationMDMTestSuite) TestBatchSetSoftwareInstallers() { s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &titlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(tm.ID))) require.Equal(t, 0, titlesResp.Count) require.Len(t, titlesResp.SoftwareTitles, 0) - } func (s *integrationEnterpriseTestSuite) TestSoftwareInstallerNewInstallRequestPlatformValidation() {