Feature 6975: Populate vendor column in software inventory (#7297)

- Populate 'software.vendor' when ingesting software from Windows hosts.
- Increate width of 'software.vendor'.

changes/feature-6975-populate-vendor-for-windows-software

@@ -0,0 +1 @@
+* Populate the vendor column on software ingested from Windows systems.

server/datastore/mysql/migrations/tables/20220818101352_ChangeSoftwareVendorWidth.go

@@ -0,0 +1,78 @@
+package tables
+
+import (
+	"database/sql"
+
+	"github.com/pkg/errors"
+)
+
+func init() {
+	MigrationClient.AddMigration(Up_20220818101352, Down_20220818101352)
+}
+
+func Up_20220818101352(tx *sql.Tx) error {
+	logger.Info.Println("Increasing width of software.vendor...")
+
+	//-----------------
+	// Add temp column.
+	//-----------------
+	if _, err := tx.Exec(
+		`ALTER TABLE software ADD COLUMN vendor_wide varchar(114) NULL, ALGORITHM=INPLACE, LOCK=NONE`); err != nil {
+		return errors.Wrapf(err, "creating temp column for vendor")
+	}
+
+	//---------------------
+	// Add uniq constraint
+	//---------------------
+	if _, err := tx.Exec(
+		"ALTER TABLE software ADD constraint unq_name UNIQUE (name, version, source, `release`, vendor_wide, arch)"); err != nil {
+		return errors.Wrapf(err, "adding new uniquess constraint")
+	}
+
+	//------------------
+	// Update in batches
+	//------------------
+	const updateStmt = `UPDATE software SET vendor_wide = vendor WHERE vendor_wide IS NULL LIMIT 500`
+	for {
+		res, err := tx.Exec(updateStmt)
+		if err != nil {
+			return errors.Wrapf(err, "updating temp vendor column")
+		}
+		affected, err := res.RowsAffected()
+		if err != nil {
+			return errors.Wrapf(err, "updating temp vendor column")
+		}
+		if affected == 0 {
+			break
+		}
+	}
+
+	//----------------
+	// Drop old index
+	//----------------
+	if _, err := tx.Exec(`ALTER TABLE software DROP KEY name`); err != nil {
+		return errors.Wrapf(err, "dropping old index")
+	}
+
+	//------------------
+	// Rename old column
+	//------------------
+	if _, err := tx.Exec(`ALTER TABLE software CHANGE vendor vendor_old varchar(32) DEFAULT '' NOT NULL, ALGORITHM=INPLACE, LOCK=NONE`); err != nil {
+		return errors.Wrapf(err, "dropping old column")
+	}
+
+	// ---------------
+	// Rename column
+	// ---------------
+	if _, err := tx.Exec(
+		`ALTER TABLE software CHANGE vendor_wide vendor varchar(114) DEFAULT '' NOT NULL, ALGORITHM=INPLACE, LOCK=NONE`); err != nil {
+		return errors.Wrapf(err, "dropping old column")
+	}
+
+	logger.Info.Println("Done increasing width of software.vendor...")
+	return nil
+}
+
+func Down_20220818101352(tx *sql.Tx) error {
+	return nil
+}

server/datastore/mysql/migrations/tables/20220818101352_ChangeSoftwareVendorWidth_test.go

@@ -0,0 +1,38 @@
+package tables
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUp_20220818101352(t *testing.T) {
+	db := applyUpToPrev(t)
+
+	_, err := db.Exec(`INSERT INTO software (name, version, source, bundle_identifier, vendor, arch)
+	VALUES 
+	('zchunk-libs', '1.2.1', 'rpm_packages', '', 'Fedora Project', 'x86_64'), 
+	('word', '1.2.1', 'rpm_packages', '', 'Fake MS', 'x86_64'),
+	('excel', '1.2.1', 'rpm_packages', '', '', 'x86_64')
+	`)
+	require.NoError(t, err)
+
+	// Apply current migration.
+	applyNext(t, db)
+
+	// Check all old vendors are still there
+	var vendors []string
+	err = db.Select(&vendors, `SELECT vendor FROM software`)
+	require.NoError(t, err)
+	require.ElementsMatch(t, []string{"Fedora Project", "Fake MS", ""}, vendors)
+
+	// Check we can store a longer vendors
+	randVendor := `
+	oFZTwTV5WxJt02EVHEBcnhLzuJ8wnxKwfbabPWy7yTSiQbabEcAGDVmoXKZEZJLWObGD0cVfYptInHYgKjtDeDsBh2a8669EnyAqyBECXbFjSh`
+
+	_, err = db.Exec(
+		`INSERT INTO software (name, version, source, bundle_identifier, vendor, arch) VALUES  ('zchunk-libs', '1.2.1', 'rpm_packages', '', ?, 'x86_64')`,
+		randVendor,
+	)
+	require.NoError(t, err)
+}

server/fleet/software.go

@@ -34,6 +34,10 @@ type CVEMeta struct {
 	Published *time.Time `db:"published"`
 }
 
+// Must be kept in sync with the vendor column definition.
+const SoftwareVendorMaxLength = 114
+const SoftwareVendorMaxLengthFmt = "%.111s..."
+
 // Software is a named and versioned piece of software installed on a device.
 type Software struct {
 	ID uint `json:"id" db:"id"`
@@ -51,6 +55,12 @@ type Software struct {
 	Release string `json:"release,omitempty" db:"release"`
 	// Vendor is the supplier of the software (e.g. "CentOS").
 	Vendor string `json:"vendor,omitempty" db:"vendor"`
+
+	// TODO: Remove this as part of the clean up of https://github.com/fleetdm/fleet/pull/7297
+	// DO NOT USE THIS, use 'Vendor' instead. We had to 'recreate' the vendor column because we
+	// needed to make it wider - the old column was left and renamed to 'vendor_old'
+	VendorOld string `json:"-" db:"vendor_old"`
+
 	// Arch is the architecture of the software (e.g. "x86_64").
 	Arch string `json:"arch,omitempty" db:"arch"`
 

server/service/osquery_utils/queries.go

@@ -9,6 +9,7 @@ import (
 	"strconv"
 	"strings"
 	"time"
+	"unicode/utf8"
 
 	"github.com/fleetdm/fleet/v4/server/config"
 	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
@@ -581,57 +582,57 @@ SELECT
   name AS name,
   version AS version,
   'Program (Windows)' AS type,
-  'programs' AS source
+  'programs' AS source,
+  publisher AS vendor
 FROM programs
 UNION
 SELECT
   name AS name,
   version AS version,
   'Package (Python)' AS type,
-  'python_packages' AS source
+  'python_packages' AS source,
+  '' AS vendor
 FROM python_packages
 UNION
 SELECT
   name AS name,
   version AS version,
   'Browser plugin (IE)' AS type,
-  'ie_extensions' AS source
+  'ie_extensions' AS source,
+  '' AS vendor
 FROM ie_extensions
 UNION
 SELECT
   name AS name,
   version AS version,
   'Browser plugin (Chrome)' AS type,
-  'chrome_extensions' AS source
+  'chrome_extensions' AS source,
+  '' AS vendor
 FROM cached_users CROSS JOIN chrome_extensions USING (uid)
 UNION
 SELECT
   name AS name,
   version AS version,
   'Browser plugin (Firefox)' AS type,
-  'firefox_addons' AS source
+  'firefox_addons' AS source,
+  '' AS vendor
 FROM cached_users CROSS JOIN firefox_addons USING (uid)
 UNION
 SELECT
   name AS name,
   version AS version,
   'Package (Chocolatey)' AS type,
-  'chocolatey_packages' AS source
+  'chocolatey_packages' AS source,
+  '' AS vendor
 FROM chocolatey_packages
 UNION
 SELECT
   name AS name,
   version AS version,
   'Package (Atom)' AS type,
-  'atom_packages' AS source
-FROM cached_users CROSS JOIN atom_packages USING (uid)
-UNION
-SELECT
-  name AS name,
-  version AS version,
-  'Package (Python)' AS type,
-  'python_packages' AS source
-FROM python_packages;
+  'atom_packages' AS source,
+  '' AS vendor
+FROM cached_users CROSS JOIN atom_packages USING (uid);
 `),
 	Platforms:        []string{"windows"},
 	DirectIngestFunc: directIngestSoftware,
@@ -861,6 +862,8 @@ func directIngestSoftware(ctx context.Context, logger log.Logger, host *fleet.Ho
 		version := row["version"]
 		source := row["source"]
 		bundleIdentifier := row["bundle_identifier"]
+		vendor := row["vendor"]
+
 		if name == "" {
 			level.Debug(logger).Log(
 				"msg", "host reported software with empty name",
@@ -895,6 +898,11 @@ func directIngestSoftware(ctx context.Context, logger log.Logger, host *fleet.Ho
 			}
 		}
 
+		// Check whether the vendor is longer than the max allowed width and if so, truncate it.
+		if utf8.RuneCountInString(vendor) >= fleet.SoftwareVendorMaxLength {
+			vendor = fmt.Sprintf(fleet.SoftwareVendorMaxLengthFmt, vendor)
+		}
+
 		s := fleet.Software{
 			Name:             name,
 			Version:          version,
@@ -902,7 +910,7 @@ func directIngestSoftware(ctx context.Context, logger log.Logger, host *fleet.Ho
 			BundleIdentifier: bundleIdentifier,
 
 			Release: row["release"],
-			Vendor:  row["vendor"],
+			Vendor:  vendor,
 			Arch:    row["arch"],
 		}
 		if !lastOpenedAt.IsZero() {

server/service/osquery_utils/queries_test.go

@@ -657,3 +657,58 @@ func TestDangerousReplaceQuery(t *testing.T) {
 	queries = GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true}}, config.FleetConfig{})
 	assert.Equal(t, originalQuery, queries["users"].Query)
 }
+
+func TestDirectIngestSoftware(t *testing.T) {
+	ds := new(mock.Store)
+
+	t.Run("vendor gets truncated", func(t *testing.T) {
+		for i, tc := range []struct {
+			data     []map[string]string
+			expected string
+		}{
+			{
+				data: []map[string]string{
+					{
+						"name":              "Software 1",
+						"version":           "12.5",
+						"source":            "My backyard",
+						"bundle_identifier": "",
+						"vendor":            "Fleet",
+					},
+				},
+				expected: "Fleet",
+			},
+			{
+				data: []map[string]string{
+					{
+						"name":              "Software 1",
+						"version":           "12.5",
+						"source":            "My backyard",
+						"bundle_identifier": "",
+						"vendor":            `oFZTwTV5WxJt02EVHEBcnhLzuJ8wnxKwfbabPWy7yTSiQbabEcAGDVmoXKZEZJLWObGD0cVfYptInHYgKjtDeDsBh2a8669EnyAqyBECXbFjSh1111`,
+					},
+				},
+				expected: `oFZTwTV5WxJt02EVHEBcnhLzuJ8wnxKwfbabPWy7yTSiQbabEcAGDVmoXKZEZJLWObGD0cVfYptInHYgKjtDeDsBh2a8669EnyAqyBECXbFjSh1...`,
+			},
+		} {
+			ds.UpdateHostSoftwareFunc = func(ctx context.Context, hostID uint, software []fleet.Software) error {
+				require.Len(t, software, 1)
+				require.Equal(t, tc.expected, software[0].Vendor)
+				return nil
+			}
+
+			err := directIngestSoftware(
+				context.Background(),
+				log.NewNopLogger(),
+				&fleet.Host{ID: uint(i)},
+				ds,
+				tc.data,
+				false,
+			)
+
+			require.NoError(t, err)
+			require.True(t, ds.UpdateHostSoftwareFuncInvoked)
+			ds.UpdateHostSoftwareFuncInvoked = false
+		}
+	})
+}