From 351f40230a60c10909f1dd40fc549fa3181f5b80 Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Wed, 19 Feb 2025 14:44:01 -0600 Subject: [PATCH] Add osquery ingestion for host certificates feature (#26426) --- .../Contributing/Understanding-host-vitals.md | 17 +++ server/datastore/mysql/host_certificates.go | 12 +-- server/fleet/host_certificates.go | 36 ++++++- server/fleet/host_certificates_test.go | 100 ++++++++++++++++++ server/service/osquery_utils/queries.go | 68 ++++++++++++ server/service/osquery_utils/queries_test.go | 57 +++++++++- 6 files changed, 280 insertions(+), 10 deletions(-) create mode 100644 server/fleet/host_certificates_test.go diff --git a/docs/Contributing/Understanding-host-vitals.md b/docs/Contributing/Understanding-host-vitals.md index f4b6a1d362..42531f793d 100644 --- a/docs/Contributing/Understanding-host-vitals.md +++ b/docs/Contributing/Understanding-host-vitals.md @@ -17,6 +17,23 @@ SELECT 1 FROM osquery_registry WHERE active = true AND registry = 'table' AND na SELECT serial_number, cycle_count, designed_capacity, max_capacity FROM battery ``` +## certificates_darwin + +- Platforms: darwin + +- Query: +```sql +SELECT + ca, common_name, subject, issuer, + key_algorithm, key_strength, key_usage, signing_algorithm, + not_valid_after, not_valid_before, + serial, sha1 + FROM + certificates + WHERE + path = '/Library/Keychains/System.keychain'; +``` + ## chromeos_profile_user_info - Platforms: chrome diff --git a/server/datastore/mysql/host_certificates.go b/server/datastore/mysql/host_certificates.go index 19dada7419..c4ab2d41b6 100644 --- a/server/datastore/mysql/host_certificates.go +++ b/server/datastore/mysql/host_certificates.go @@ -35,7 +35,7 @@ func (ds *Datastore) UpdateHostCertificates(ctx context.Context, hostID uint, ce // infrequently and they will be eventually consistent existingCerts, _, err := listHostCertsDB(ctx, ds.reader(ctx), hostID, fleet.ListOptions{}) // requesting unpaginated results with default limit of 1 million if err != nil { - return fmt.Errorf("list host certs for update: %w", err) + return ctxerr.Wrap(ctx, err, "list host certificates for update") } existingBySHA1 := make(map[string]*fleet.HostCertificateRecord, len(existingCerts)) for _, ec := range existingCerts { @@ -115,7 +115,7 @@ WHERE var certs []*fleet.HostCertificateRecord if err := sqlx.SelectContext(ctx, tx, &certs, stmtPaged, args...); err != nil { - return nil, nil, err + return nil, nil, ctxerr.Wrap(ctx, err, "selecting host certificates") } var metaData *fleet.PaginationMetadata @@ -169,10 +169,10 @@ INSERT INTO host_certificates ( cert.IssuerCountry, cert.IssuerOrganization, cert.IssuerOrganizationalUnit, cert.IssuerCommonName) } - stmt = fmt.Sprintf(stmt, strings.Trim(strings.Join(placeholders, ","), ",")) + stmt = fmt.Sprintf(stmt, strings.Join(placeholders, ",")) if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { - return err + return ctxerr.Wrap(ctx, err, "inserting host certificates") } return nil @@ -186,11 +186,11 @@ func softDeleteHostCertsDB(ctx context.Context, tx sqlx.ExtContext, hostID uint, stmt := `UPDATE host_certificates SET deleted_at = NOW(6) WHERE host_id = ? AND id IN (?)` stmt, args, err := sqlx.In(stmt, hostID, toDelete) if err != nil { - return err + return ctxerr.Wrap(ctx, err, "building soft delete query") } if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { - return err + return ctxerr.Wrap(ctx, err, "soft deleting host certificates") } return nil diff --git a/server/fleet/host_certificates.go b/server/fleet/host_certificates.go index 97b26c38c5..898fb1e10b 100644 --- a/server/fleet/host_certificates.go +++ b/server/fleet/host_certificates.go @@ -5,6 +5,7 @@ import ( "crypto/x509" "errors" "fmt" + "strings" "time" ) @@ -139,8 +140,39 @@ type MDMAppleErrorChainItem struct { // // See https://osquery.io/schema/5.15.0/#certificates func ExtractDetailsFromOsqueryDistinguishedName(str string) (*HostCertificateNameDetails, error) { - // TODO - return nil, errors.New("not implemented") + str = strings.TrimSpace(str) + str = strings.Trim(str, "/") + + if !strings.Contains(str, "/") { + return nil, errors.New("invalid format, wrong separator") + } + + parts := strings.Split(str, "/") + + var details HostCertificateNameDetails + for _, part := range parts { + kv := strings.Split(part, "=") + if len(kv) != 2 { + return nil, errors.New("invalid distinguished name, wrong key value pair format") + } + + if len(kv[1]) == 0 { + return nil, errors.New("invalid distinguished name, missing value") + } + + switch strings.ToUpper(kv[0]) { + case "C": + details.Country = strings.Trim(kv[1], " ") + case "O": + details.Organization = strings.Trim(kv[1], " ") + case "OU": + details.OrganizationalUnit = strings.Trim(kv[1], " ") + case "CN": + details.CommonName = strings.Trim(kv[1], " ") + } + } + + return &details, nil } func firstOrEmpty(s []string) string { diff --git a/server/fleet/host_certificates_test.go b/server/fleet/host_certificates_test.go new file mode 100644 index 0000000000..e6c6219e14 --- /dev/null +++ b/server/fleet/host_certificates_test.go @@ -0,0 +1,100 @@ +package fleet + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestExtractHostCertificateNameDetails(t *testing.T) { + expected := HostCertificateNameDetails{ + Country: "US", + Organization: "Fleet Device Management Inc.", + OrganizationalUnit: "Fleet Device Management Inc.", + CommonName: "FleetDM", + } + + cases := []struct { + name string + input string + expected *HostCertificateNameDetails + err bool + }{ + { + name: "valid", + input: "/C=US/O=Fleet Device Management Inc./OU=Fleet Device Management Inc./CN=FleetDM", + expected: &expected, + }, + { + name: "valid with different order", + input: "/O=Fleet Device Management Inc./OU=Fleet Device Management Inc./CN=FleetDM/C=US", + expected: &expected, + }, + { + name: "valid with missing key", + input: "/C=US/O=Fleet Device Management Inc./CN=FleetDM ", + expected: &HostCertificateNameDetails{ + Country: "US", + Organization: "Fleet Device Management Inc.", + OrganizationalUnit: "", + CommonName: "FleetDM", + }, + }, + { + name: "valid with additional keyr", + input: "/C=US/O=Fleet Device Management Inc./OU=Fleet Device Management Inc./CN=FleetDM/L=SomeCity", + expected: &expected, + }, + { + name: "invalid format with extra slash", + input: "/C=US/O=Fleet Device Management Inc./OU=Fleet Device Management Inc./CN=FleetDM/invalid", + err: true, + }, + { + name: "invalid format with wrong separator", + input: "C=US,O=Fleet Device Management Inc.,OU=Fleet Device Management Inc.,CN=FleetDM", + err: true, + }, + { + name: "invalid format with extra equal", + input: "/C=US=/O=Fleet Device Management Inc./OU=Fleet Device Management Inc./CN=FleetDM", + err: true, + }, + { + name: "invalid format with malformed key values", + input: "/C=US/O/OU=Fleet Device Management Inc./=/CN=FleetDM", + err: true, + }, + { + name: "empty", + input: "", + err: true, + }, + { + name: "missing value", + input: "/C=US/O=Fleet Device Management Inc./OU=Fleet Device Management Inc./CN=", + err: true, + }, + { + name: "missing first slash", + input: "C=US/O=Fleet Device Management Inc./OU=Fleet Device Management Inc./CN=FleetDM", + expected: &expected, + }, + { + name: "trailing slash", + input: "/C=US/O=Fleet Device Management Inc./OU=Fleet Device Management Inc./CN=FleetDM/", + expected: &expected, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + actual, err := ExtractDetailsFromOsqueryDistinguishedName(tc.input) + if tc.err { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tc.expected, actual) + } + }) + } +} diff --git a/server/service/osquery_utils/queries.go b/server/service/osquery_utils/queries.go index 0e07446036..8306776665 100644 --- a/server/service/osquery_utils/queries.go +++ b/server/service/osquery_utils/queries.go @@ -693,6 +693,20 @@ var extraDetailQueries = map[string]DetailQuery{ Platforms: []string{"windows"}, DirectIngestFunc: directIngestDiskEncryption, }, + "certificates_darwin": { + Query: ` + SELECT + ca, common_name, subject, issuer, + key_algorithm, key_strength, key_usage, signing_algorithm, + not_valid_after, not_valid_before, + serial, sha1 + FROM + certificates + WHERE + path = '/Library/Keychains/System.keychain';`, + Platforms: []string{"darwin"}, + DirectIngestFunc: directIngestHostCertificates, + }, } // mdmQueries are used by the Fleet server to compliment certain MDM @@ -2376,3 +2390,57 @@ func directIngestWindowsProfiles( } return microsoft_mdm.VerifyHostMDMProfiles(ctx, logger, ds, host, rawResponse) } + +func directIngestHostCertificates( + ctx context.Context, + logger log.Logger, + host *fleet.Host, + ds fleet.Datastore, + rows []map[string]string, +) error { + if len(rows) == 0 { + // if there are no results, it probably may indicate a problem so we log it + level.Debug(logger).Log("component", "service", "method", "directIngestHostCertificates", "msg", "no rows returned", "host_id", host.ID) + return nil + } + + certs := make([]*fleet.HostCertificateRecord, 0, len(rows)) + for _, row := range rows { + csum, err := hex.DecodeString(row["sha1"]) + if err != nil { + return ctxerr.Wrap(ctx, err, "directIngestHostCertificates: decoding sha1") + } + subject, err := fleet.ExtractDetailsFromOsqueryDistinguishedName(row["subject"]) + if err != nil { + return ctxerr.Wrap(ctx, err, "directIngestHostCertificates: extracting subject details") + } + issuer, err := fleet.ExtractDetailsFromOsqueryDistinguishedName(row["issuer"]) + if err != nil { + return ctxerr.Wrap(ctx, err, "directIngestHostCertificates: extracting issuer details") + } + + certs = append(certs, &fleet.HostCertificateRecord{ + HostID: host.ID, + SHA1Sum: csum, + NotValidAfter: time.Unix(cast.ToInt64(row["not_valid_after"]), 0).UTC(), + NotValidBefore: time.Unix(cast.ToInt64(row["not_valid_before"]), 0).UTC(), + CertificateAuthority: cast.ToBool(row["ca"]), + CommonName: row["common_name"], + KeyAlgorithm: row["key_algorithm"], + KeyStrength: cast.ToInt(row["key_strength"]), + KeyUsage: row["key_usage"], + Serial: row["serial"], + SigningAlgorithm: row["signing_algorithm"], + SubjectCountry: subject.Country, + SubjectOrganizationalUnit: subject.OrganizationalUnit, + SubjectOrganization: subject.Organization, + SubjectCommonName: subject.CommonName, + IssuerCountry: issuer.Country, + IssuerOrganizationalUnit: issuer.OrganizationalUnit, + IssuerOrganization: issuer.Organization, + IssuerCommonName: issuer.CommonName, + }) + } + + return ds.UpdateHostCertificates(ctx, host.ID, certs) +} diff --git a/server/service/osquery_utils/queries_test.go b/server/service/osquery_utils/queries_test.go index 8292832fd6..a2486fdcfe 100644 --- a/server/service/osquery_utils/queries_test.go +++ b/server/service/osquery_utils/queries_test.go @@ -290,13 +290,14 @@ func TestGetDetailQueries(t *testing.T) { "disk_encryption_linux", "disk_encryption_windows", "chromeos_profile_user_info", + "certificates_darwin", } require.Len(t, queriesNoConfig, len(baseQueries)) sortedKeysCompare(t, queriesNoConfig, baseQueries) queriesWithoutWinOSVuln := GetDetailQueries(context.Background(), config.FleetConfig{Vulnerabilities: config.VulnerabilitiesConfig{DisableWinOSVulnerabilities: true}}, nil, nil) - require.Len(t, queriesWithoutWinOSVuln, 25) + require.Len(t, queriesWithoutWinOSVuln, 26) queriesWithUsers := GetDetailQueries(context.Background(), config.FleetConfig{App: config.AppConfig{EnableScheduledQueryStats: true}}, nil, &fleet.Features{EnableHostUsers: true}) qs := baseQueries @@ -1530,7 +1531,8 @@ func TestDirectIngestDiskEncryptionKeyDarwin(t *testing.T) { } ds.SetOrUpdateHostDiskEncryptionKeyFunc = func(ctx context.Context, incomingHost *fleet.Host, encryptedBase64Key, clientError string, - decryptable *bool) error { + decryptable *bool, + ) error { if base64.StdEncoding.EncodeToString([]byte(wantKey)) != encryptedBase64Key { return errors.New("key mismatch") } @@ -2219,6 +2221,57 @@ func TestIngestNetworkInterface(t *testing.T) { }) } +func TestDirectIngestHostCertificates(t *testing.T) { + ds := new(mock.Store) + ctx := context.Background() + logger := log.NewNopLogger() + host := &fleet.Host{ID: 1} + + row1 := map[string]string{ + "ca": "0", + "common_name": "Cert 1 Common Name", + "issuer": "/C=US/O=Issuer 1 Inc./CN=Issuer 1 Common Name", + "subject": "/C=US/O=Subject 1 Inc./OU=Subject 1 Org Unit/CN=Subject 1 Common Name", + "key_algorithm": "rsaEncryption", + "key_strength": "2048", + "key_usage": "Data Encipherment, Key Encipherment, Digital Signature", + "serial": "123abc", + "signing_algorithm": "sha256WithRSAEncryption", + "not_valid_after": "1822755797", + "not_valid_before": "1770228826", + "sha1": "9c1e9c00d8120c1a9d96274d2a17c38ffa30fd31", + } + + ds.UpdateHostCertificatesFunc = func(ctx context.Context, hostID uint, certs []*fleet.HostCertificateRecord) error { + require.Equal(t, host.ID, hostID) + require.Len(t, certs, 1) + require.Equal(t, "9c1e9c00d8120c1a9d96274d2a17c38ffa30fd31", hex.EncodeToString(certs[0].SHA1Sum)) + require.Equal(t, "Cert 1 Common Name", certs[0].CommonName) + require.Equal(t, "Subject 1 Common Name", certs[0].SubjectCommonName) + require.Equal(t, "Subject 1 Inc.", certs[0].SubjectOrganization) + require.Equal(t, "Subject 1 Org Unit", certs[0].SubjectOrganizationalUnit) + require.Equal(t, "US", certs[0].SubjectCountry) + require.Equal(t, "Issuer 1 Common Name", certs[0].IssuerCommonName) + require.Equal(t, "Issuer 1 Inc.", certs[0].IssuerOrganization) + require.Empty(t, certs[0].IssuerOrganizationalUnit) + require.Equal(t, "US", certs[0].IssuerCountry) + require.Equal(t, "rsaEncryption", certs[0].KeyAlgorithm) + require.Equal(t, 2048, certs[0].KeyStrength) + require.Equal(t, "Data Encipherment, Key Encipherment, Digital Signature", certs[0].KeyUsage) + require.Equal(t, "123abc", certs[0].Serial) + require.Equal(t, "sha256WithRSAEncryption", certs[0].SigningAlgorithm) + require.Equal(t, int64(1822755797), certs[0].NotValidAfter.Unix()) + require.Equal(t, int64(1770228826), certs[0].NotValidBefore.Unix()) + require.False(t, certs[0].CertificateAuthority) + + return nil + } + + err := directIngestHostCertificates(ctx, logger, host, ds, []map[string]string{row1}) + require.NoError(t, err) + require.True(t, ds.UpdateHostCertificatesFuncInvoked) +} + func TestGenerateSQLForAllExists(t *testing.T) { // Combine two queries query1 := "SELECT 1 WHERE foo = bar"