Add "ExcludeFleetMaintainedApps" option to software titles query (#25649)

for #25427 

# Checklist for submitter

<!-- Note that API documentation changes are now addressed by the
product design team. -->

- [X] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/Committing-Changes.md#changes-files)
for more information.
- [X] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements)

This PR adds a new `ExcludeFleetMaintainedApps` option to the
ListSoftwareTitles datastore method, and the equivalent
`exclude_fleet_maintained_apps` to the `GET
/api/v1/fleet/software/titles` API.

The new functionality works by doing a left join from the
`software_titles` table to the `fleet_library_apps` table by bundle
identifier, and excluding any rows that are present in the
`fleet_library_apps` table.

New tests verify that the filtering works as expected and doesn't
interfere with other functions of the method.

changes/25427-allow-excluding-fma-from-software-titles

@@ -0,0 +1 @@
+- Add "exclude_fleet_maintained_apps" option to `GET /api/v1/fleet/software/titles`

server/datastore/mysql/software_titles.go

@@ -410,6 +410,12 @@ GROUP BY st.id, package_self_service, package_name, package_version, package_url
 		defaultFilter += ` AND ( si.self_service = 1 OR vat.self_service = 1 ) `
 	}
 
+	// if excluding fleet maintained apps, filter out any row from software_titles
+	// that has a matching row in fleet_library_apps.
+	if opt.ExcludeFleetMaintainedApps {
+		additionalWhere += " AND NOT EXISTS ( SELECT FALSE FROM fleet_library_apps AS fla WHERE fla.bundle_identifier = st.bundle_identifier )"
+	}
+
 	stmt = fmt.Sprintf(stmt, softwareInstallersJoinCond, vppAppsJoinCond, vppAppsTeamsJoinCond, countsJoin, softwareJoin, additionalWhere, defaultFilter)
 	return stmt, args
 }

server/datastore/mysql/software_titles_test.go

@@ -29,6 +29,7 @@ func TestSoftwareTitles(t *testing.T) {
 		{"ListSoftwareTitlesInstallersOnly", testListSoftwareTitlesInstallersOnly},
 		{"ListSoftwareTitlesAvailableForInstallFilter", testListSoftwareTitlesAvailableForInstallFilter},
 		{"ListSoftwareTitlesAllTeams", testListSoftwareTitlesAllTeams},
+		{"ListSoftwareTitlesNotFleetMaintained", testListSoftwareTitlesNotFleetMaintained},
 		{"UploadedSoftwareExists", testUploadedSoftwareExists},
 		{"ListSoftwareTitlesVulnerabilityFilters", testListSoftwareTitlesVulnerabilityFilters},
 	}
@@ -639,6 +640,7 @@ func testTeamFilterSoftwareTitles(t *testing.T, ds *Datastore) {
 		InstallScript:    "echo",
 		Filename:         "installer1.pkg",
 		BundleIdentifier: "foo.bar",
+		Platform:         string(fleet.MacOSPlatform),
 		TeamID:           &team1.ID,
 		UserID:           user1.ID,
 		ValidatedLabels:  &fleet.LabelIdentsWithScope{},
@@ -865,6 +867,20 @@ func testTeamFilterSoftwareTitles(t *testing.T, ds *Datastore) {
 	require.NoError(t, err)
 	require.Len(t, titles, 1)
 	require.Equal(t, "vpp3", titles[0].Name)
+
+	// Testing with Platform filter
+	titles, count, _, err = ds.ListSoftwareTitles(
+		context.Background(), fleet.SoftwareTitleListOptions{
+			ListOptions:         fleet.ListOptions{},
+			Platform:            string(fleet.MacOSPlatform),
+			AvailableForInstall: true,
+			TeamID:              &team1.ID,
+		}, globalTeamFilter,
+	)
+	require.NoError(t, err)
+	require.Equal(t, 1, count)
+	require.Len(t, titles, 1)
+	require.Equal(t, "installer1", titles[0].Name)
 }
 
 func sortTitlesByName(titles []fleet.SoftwareTitleListResult) {
@@ -1371,6 +1387,213 @@ func testListSoftwareTitlesAllTeams(t *testing.T, ds *Datastore) {
 	}, names)
 }
 
+func testListSoftwareTitlesNotFleetMaintained(t *testing.T, ds *Datastore) {
+	ctx := context.Background()
+
+	test.CreateInsertGlobalVPPToken(t, ds)
+
+	// Add a couple of apps to the Fleet library.
+	_, err := ds.UpsertMaintainedApp(ctx, &fleet.MaintainedApp{
+		Name:    "Awesome Fleet Maintained App",
+		Token:   "abc123",
+		Version: "1.2.3",
+		// for now, maintained apps are always macOS (darwin)
+		Platform:         fleet.MacOSPlatform,
+		InstallerURL:     "http://installmycoolapp.com",
+		SHA256:           "abc123",
+		BundleIdentifier: "com.fleetmaintained_installer.xyz",
+		InstallScript:    "echo",
+		UninstallScript:  "sleep",
+	})
+	require.NoError(t, err)
+	_, err = ds.UpsertMaintainedApp(ctx, &fleet.MaintainedApp{
+		Name:    "Another Fleet Maintained App",
+		Token:   "xxxyyy",
+		Version: "5.5.5",
+		// for now, maintained apps are always macOS (darwin)
+		Platform:         fleet.MacOSPlatform,
+		InstallerURL:     "http://installmycoolapp.com",
+		SHA256:           "xyz999",
+		BundleIdentifier: "fma.com.xyz",
+		InstallScript:    "echo",
+		UninstallScript:  "sleep",
+	})
+	require.NoError(t, err)
+
+	user1 := test.NewUser(t, ds, "Alice", "alice@example.com", true)
+
+	// Create a macOS software foobar installer on "No team".
+	macOSInstallerNoTeam, _, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{
+		Title:            "foobar",
+		BundleIdentifier: "com.foo.bar",
+		Source:           "apps",
+		InstallScript:    "echo",
+		Filename:         "foobar.pkg",
+		TeamID:           nil,
+		UserID:           user1.ID,
+		ValidatedLabels:  &fleet.LabelIdentsWithScope{},
+	})
+	require.NoError(t, err)
+	require.NotZero(t, macOSInstallerNoTeam)
+
+	// Create another macOS software installer on "No team" that we'll set as a Fleet Maintained App.
+	macOSInstallerNoTeam2, _, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{
+		Title:            "fleetmaintained_installer",
+		BundleIdentifier: "com.fleetmaintained_installer.xyz",
+		Source:           "apps",
+		InstallScript:    "echo",
+		Filename:         "fleetmaintained_installer.pkg",
+		TeamID:           nil,
+		UserID:           user1.ID,
+		ValidatedLabels:  &fleet.LabelIdentsWithScope{},
+	})
+	require.NoError(t, err)
+	require.NotZero(t, macOSInstallerNoTeam2)
+
+	// Add a macOS host on "No team" with some software.
+	host := test.NewHost(t, ds, "host", "", "hostkey", "hostuuid", time.Now())
+	software := []fleet.Software{
+		{Name: "foo", Version: "0.0.1", Source: "chrome_extensions", BundleIdentifier: "foo.com.bar"},
+		{Name: "foo", Version: "0.0.3", Source: "chrome_extensions", BundleIdentifier: "foo.com.bar"},
+		{Name: "bar", Version: "0.0.3", Source: "deb_packages", BundleIdentifier: "bar.com.baz"},
+		{Name: "fma", Version: "1.2.3", Source: "deb_packages", BundleIdentifier: "fma.com.xyz"},
+	}
+	_, err = ds.UpdateHostSoftware(ctx, host.ID, software)
+	require.NoError(t, err)
+
+	// Simulate vulnerabilities cron
+	require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now()))
+	require.NoError(t, ds.ReconcileSoftwareTitles(ctx))
+	require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now()))
+
+	// List software titles for "All teams", should only return the host software titles
+	// and no installers/VPP-apps because none is installed yet.
+	titles, counts, _, err := ds.ListSoftwareTitles(
+		ctx,
+		fleet.SoftwareTitleListOptions{
+			ListOptions: fleet.ListOptions{
+				OrderKey:       "name",
+				OrderDirection: fleet.OrderAscending,
+			},
+			ExcludeFleetMaintainedApps: true,
+			TeamID:                     nil,
+		},
+		fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}},
+	)
+	require.NoError(t, err)
+	assert.EqualValues(t, 2, counts)
+	assert.Len(t, titles, 2)
+	type nameSource struct {
+		name   string
+		source string
+	}
+	names := make([]nameSource, 0, len(titles))
+	for _, title := range titles {
+		names = append(names, nameSource{name: title.Name, source: title.Source})
+	}
+	assert.ElementsMatch(t, []nameSource{
+		{name: "bar", source: "deb_packages"},
+		{name: "foo", source: "chrome_extensions"},
+	}, names)
+
+	// List software for "No team". Should list the host's software + the macOS installer.
+	titles, counts, _, err = ds.ListSoftwareTitles(
+		ctx,
+		fleet.SoftwareTitleListOptions{
+			ListOptions: fleet.ListOptions{
+				OrderKey:       "name",
+				OrderDirection: fleet.OrderAscending,
+			},
+			ExcludeFleetMaintainedApps: true,
+			TeamID:                     ptr.Uint(0),
+		},
+		fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}},
+	)
+	require.NoError(t, err)
+	assert.EqualValues(t, 3, counts)
+	assert.Len(t, titles, 3)
+	names = make([]nameSource, 0, len(titles))
+	for _, title := range titles {
+		names = append(names, nameSource{name: title.Name, source: title.Source})
+	}
+	assert.ElementsMatch(t, []nameSource{
+		{name: "bar", source: "deb_packages"},
+		{name: "foo", source: "chrome_extensions"},
+		{name: "foobar", source: "apps"},
+	}, names)
+
+	// List software for "No team", with match for non-FMA title.
+	// This should return a result.
+	titles, counts, _, err = ds.ListSoftwareTitles(
+		ctx,
+		fleet.SoftwareTitleListOptions{
+			ListOptions: fleet.ListOptions{
+				OrderKey:       "name",
+				OrderDirection: fleet.OrderAscending,
+				MatchQuery:     "foobar",
+			},
+			ExcludeFleetMaintainedApps: true,
+			TeamID:                     ptr.Uint(0),
+		},
+		fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}},
+	)
+	require.NoError(t, err)
+	assert.EqualValues(t, 1, counts)
+	assert.Len(t, titles, 1)
+	names = make([]nameSource, 0, len(titles))
+	for _, title := range titles {
+		names = append(names, nameSource{name: title.Name, source: title.Source})
+	}
+	assert.ElementsMatch(t, []nameSource{
+		{name: "foobar", source: "apps"},
+	}, names)
+
+	// List software for "No team", with match for an FMA title.
+	// This should not return a result since we're excluding FMAs.
+	titles, counts, _, err = ds.ListSoftwareTitles(
+		ctx,
+		fleet.SoftwareTitleListOptions{
+			ListOptions: fleet.ListOptions{
+				OrderKey:       "name",
+				OrderDirection: fleet.OrderAscending,
+				MatchQuery:     "fma",
+			},
+			ExcludeFleetMaintainedApps: true,
+			TeamID:                     ptr.Uint(0),
+		},
+		fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}},
+	)
+	require.NoError(t, err)
+	assert.EqualValues(t, 0, counts)
+	assert.Len(t, titles, 0)
+
+	// List software available for install on "No team". Should list "foobar" package only.
+	titles, counts, _, err = ds.ListSoftwareTitles(
+		ctx,
+		fleet.SoftwareTitleListOptions{
+			ListOptions: fleet.ListOptions{
+				OrderKey:       "name",
+				OrderDirection: fleet.OrderAscending,
+			},
+			ExcludeFleetMaintainedApps: true,
+			AvailableForInstall:        true,
+			TeamID:                     ptr.Uint(0),
+		},
+		fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}},
+	)
+	require.NoError(t, err)
+	require.EqualValues(t, 1, counts)
+	require.Len(t, titles, 1)
+
+	names = make([]nameSource, 0, len(titles))
+	for _, title := range titles {
+		names = append(names, nameSource{name: title.Name, source: title.Source})
+	}
+	assert.ElementsMatch(t, []nameSource{
+		{name: "foobar", source: "apps"},
+	}, names)
+}
+
 func testUploadedSoftwareExists(t *testing.T, ds *Datastore) {
 	ctx := context.Background()
 

server/fleet/software.go

@@ -223,15 +223,16 @@ type SoftwareTitleListOptions struct {
 	// ListOptions cannot be embedded in order to unmarshall with validation.
 	ListOptions ListOptions `url:"list_options"`
 
-	TeamID              *uint   `query:"team_id,optional"`
-	VulnerableOnly      bool    `query:"vulnerable,optional"`
-	AvailableForInstall bool    `query:"available_for_install,optional"`
-	SelfServiceOnly     bool    `query:"self_service,optional"`
-	KnownExploit        bool    `query:"exploit,optional"`
-	MinimumCVSS         float64 `query:"min_cvss_score,optional"`
-	MaximumCVSS         float64 `query:"max_cvss_score,optional"`
-	PackagesOnly        bool    `query:"packages_only,optional"`
-	Platform            string  `query:"platform,optional"`
+	TeamID                     *uint   `query:"team_id,optional"`
+	VulnerableOnly             bool    `query:"vulnerable,optional"`
+	AvailableForInstall        bool    `query:"available_for_install,optional"`
+	SelfServiceOnly            bool    `query:"self_service,optional"`
+	KnownExploit               bool    `query:"exploit,optional"`
+	MinimumCVSS                float64 `query:"min_cvss_score,optional"`
+	MaximumCVSS                float64 `query:"max_cvss_score,optional"`
+	PackagesOnly               bool    `query:"packages_only,optional"`
+	Platform                   string  `query:"platform,optional"`
+	ExcludeFleetMaintainedApps bool    `query:"exclude_fleet_maintained_apps,optional"`
 }
 
 type HostSoftwareTitleListOptions struct {