Elgato_dark/fleet
1
0
Fork 0
mirror of https://github.com/fleetdm/fleet synced 2026-04-21 13:37:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 91

add a test that checks collation on new migrations (#29309)

Browse source 
> closes #26403

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

<!-- Note that API documentation changes are now addressed by the
product design team. -->

- [x] If database migrations are included, checked table schema to
confirm autoupdate
- For database migrations:
- [x] Ensured the correct collation is explicitly set for character
columns (`COLLATE utf8mb4_unicode_ci`).
- [x] Added/updated automated tests
- [x] Manual QA for all new/changed functionality
This commit is contained in:
Jahziel Villasana-Espinoza 2025-05-29 17:00:30 -04:00 committed by GitHub
parent dfb0b53a24
commit 9d2b07f76f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
65 changed files with 271 additions and 154 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
.github/workflows/test-db-changes.yml vendored
View file

@ -72,20 +72,6 @@ jobs:
index=$((index+1))
done
- name: Prevent hosts foreign keys
run: |
# grep exits with an error code if it doesn't find a match, so this condition
# is only true if it a) finds a matching migrations file in the diff, and b)
# finds an FK to hosts in one of the migrations files.
#
# grep prints the matches, which will help figure out where those references are.
if git diff --name-only origin/main | grep "migrations/" | xargs grep -i -E 'references\s*hosts\s*\(\s*id\s*\)' ; then
echo "❌ fail: hosts foreign keys are not allowed"
echo "Ref: https://github.com/fleetdm/fleet/blob/main/handbook/engineering/scaling-fleet.md#foreign-keys-and-locking"
exit 1
fi
- name: Install Go
uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
with:
@ -112,3 +98,16 @@ jobs:
echo "please run 'make test-schema' and commit the changes"
exit 1
fi
- name: Prevent hosts foreign keys
run: |
# grep exits with an error code if it doesn't find a match, so this condition
# is only true if it a) finds a matching migrations file in the diff, and b)
# finds an FK to hosts in one of the migrations files.
#
# grep prints the matches, which will help figure out where those references are.
if git diff --name-only origin/main | grep "migrations/" | xargs grep -i -E 'references\s*hosts\s*\(\s*id\s*\)' ; then
echo "❌ fail: hosts foreign keys are not allowed"
echo "Ref: https://github.com/fleetdm/fleet/blob/main/handbook/engineering/scaling-fleet.md#foreign-keys-and-locking"
exit 1
fi

2
changes/26403-db-collation Normal file
View file

@ -0,0 +1,2 @@
- Updated migrations to use the `utf8mb4_unicode_ci` collation across all tables and added a test to
validate that new migrations use this collation.

38
server/datastore/mysql/migrations/tables/20161118212528_CreateTableHosts.go
View file

@ -12,39 +12,39 @@ func Up_20161118212528(tx *sql.Tx) error {
_, err := tx.Exec(
"CREATE TABLE `hosts` (" +
"`id` int(10) unsigned NOT NULL AUTO_INCREMENT," +
"`osquery_host_id` varchar(255) NOT NULL," +
"`osquery_host_id` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL," +
"`created_at` timestamp DEFAULT CURRENT_TIMESTAMP," +
"`updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP," +
"`deleted_at` timestamp NULL DEFAULT NULL," +
"`deleted` tinyint(1) NOT NULL DEFAULT FALSE," +
"`detail_update_time` timestamp NULL DEFAULT NULL," +
"`node_key` varchar(255) DEFAULT NULL," +
"`host_name` varchar(255) NOT NULL DEFAULT ''," +
"`uuid` varchar(255) NOT NULL DEFAULT ''," +
"`platform` varchar(255) NOT NULL DEFAULT ''," +
"`osquery_version` varchar(255) NOT NULL DEFAULT ''," +
"`os_version` varchar(255) NOT NULL DEFAULT ''," +
"`build` varchar(255) NOT NULL DEFAULT ''," +
"`platform_like` varchar(255) NOT NULL DEFAULT ''," +
"`code_name` varchar(255) NOT NULL DEFAULT ''," +
"`node_key` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL," +
"`host_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''," +
"`uuid` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''," +
"`platform` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''," +
"`osquery_version` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''," +
"`os_version` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''," +
"`build` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''," +
"`platform_like` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''," +
"`code_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''," +
"`uptime` bigint(20) NOT NULL DEFAULT 0," +
"`physical_memory` bigint(20) NOT NULL DEFAULT 0," +
"`cpu_type` varchar(255) NOT NULL DEFAULT ''," +
"`cpu_subtype` varchar(255) NOT NULL DEFAULT ''," +
"`cpu_brand` varchar(255) NOT NULL DEFAULT ''," +
"`cpu_type` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''," +
"`cpu_subtype` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''," +
"`cpu_brand` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''," +
"`cpu_physical_cores` int NOT NULL DEFAULT 0," +
"`cpu_logical_cores` int NOT NULL DEFAULT 0," +
"`hardware_vendor` varchar(255) NOT NULL DEFAULT ''," +
"`hardware_model` varchar(255) NOT NULL DEFAULT ''," +
"`hardware_version` varchar(255) NOT NULL DEFAULT ''," +
"`hardware_serial` varchar(255) NOT NULL DEFAULT ''," +
"`computer_name` varchar(255) NOT NULL DEFAULT ''," +
"`hardware_vendor` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''," +
"`hardware_model` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''," +
"`hardware_version` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''," +
"`hardware_serial` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''," +
"`computer_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''," +
"`primary_ip_id` INT(10) UNSIGNED DEFAULT NULL, " +
"PRIMARY KEY (`id`)," +
"UNIQUE KEY `idx_host_unique_nodekey` (`node_key`)," +
"UNIQUE KEY `idx_osquery_host_id` (`osquery_host_id`)," +
"FULLTEXT KEY `hosts_search` (`host_name`)" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8;",
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;",
)
return err
}

2
server/datastore/mysql/migrations/tables/20161118212538_CreateTableInvites.go
View file

@ -25,7 +25,7 @@ func Up_20161118212538(tx *sql.Tx) error {
"PRIMARY KEY (`id`)," +
"UNIQUE KEY `idx_invite_unique_email` (`email`)," +
"UNIQUE KEY `idx_invite_unique_key` (`token`)" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8;",
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;",
)
return err
}

2
server/datastore/mysql/migrations/tables/20161118212557_CreateTableLabels.go
View file

@ -27,7 +27,7 @@ func Up_20161118212557(tx *sql.Tx) error {
"PRIMARY KEY (`id`)," +
"UNIQUE KEY `idx_label_unique_name` (`name`)," +
"FULLTEXT KEY `labels_search` (`name`)" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8;",
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;",
)
return err
}

2
server/datastore/mysql/migrations/tables/20161118212613_CreateTableScheduledQueries.go
View file

@ -25,7 +25,7 @@ func Up_20161118212613(tx *sql.Tx) error {
"`version` varchar(255) DEFAULT NULL," +
"`shard` int(10) unsigned DEFAULT NULL," +
"PRIMARY KEY (`id`)" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8;",
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;",
)
return err
}

2
server/datastore/mysql/migrations/tables/20161118212630_CreateTablePacks.go
View file

@ -23,7 +23,7 @@ func Up_20161118212630(tx *sql.Tx) error {
"`created_by` int(10) unsigned DEFAULT NULL," +
"PRIMARY KEY (`id`)," +
"UNIQUE KEY `idx_pack_unique_name` (`name`)" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8;",
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;",
)
return err
}

2
server/datastore/mysql/migrations/tables/20161118212641_CreateTablePasswordResetRequests.go
View file

@ -18,7 +18,7 @@ func Up_20161118212641(tx *sql.Tx) error {
"`user_id` int(10) unsigned NOT NULL," +
"`token` varchar(1024) NOT NULL," +
"PRIMARY KEY (`id`)" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8;",
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;",
)
return err
}

2
server/datastore/mysql/migrations/tables/20161118212649_CreateTableUsers.go
View file

@ -29,7 +29,7 @@ func Up_20161118212649(tx *sql.Tx) error {
"PRIMARY KEY (`id`)," +
"UNIQUE KEY `idx_user_unique_username` (`username`)," +
"UNIQUE KEY `idx_user_unique_email` (`email`)" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8;",
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;",
)
return err
}

2
server/datastore/mysql/migrations/tables/20161118212656_CreateTableSessions.go
View file

@ -18,7 +18,7 @@ func Up_20161118212656(tx *sql.Tx) error {
"`key` varchar(255) NOT NULL," +
"PRIMARY KEY (`id`)," +
"UNIQUE KEY `idx_session_unique_key` (`key`)" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8;",
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;",
)
return err
}

2
server/datastore/mysql/migrations/tables/20161118212758_CreateTableQueries.go
View file

@ -23,7 +23,7 @@ func Up_20161118212758(tx *sql.Tx) error {
"`author_id` int(10) unsigned DEFAULT NULL," +
"PRIMARY KEY (`id`)," +
"FOREIGN KEY (`author_id`) REFERENCES `users`(`id`) ON DELETE SET NULL" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8;",
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;",
)
return err
}

2
server/datastore/mysql/migrations/tables/20161128234849_CreateTableNetworkInterfaces.go
View file

@ -35,7 +35,7 @@ func Up_20161128234849(tx *sql.Tx) error {
"ON DELETE CASCADE, " +
"FULLTEXT KEY `ip_address_search` (`ip_address`)," +
"UNIQUE KEY `idx_network_interfaces_unique_ip_host_intf` (`ip_address`, `host_id`, `interface`)" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8;",
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;",
)
return err
}

2
server/datastore/mysql/migrations/tables/20170124230432_CreateTableEmailChanges.go
View file

@ -16,7 +16,7 @@ func Up_20170124230432(tx *sql.Tx) error {
"UNIQUE KEY `idx_unique_email_changes_token` (`token`) USING BTREE, " +
"KEY `fk_email_changes_users` (`user_id`), " +
"CONSTRAINT `fk_email_changes_users` FOREIGN KEY (`user_id`) REFERENCES `users` (`id`) ON DELETE CASCADE " +
") ENGINE=InnoDB DEFAULT CHARSET=utf8;"
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"
_, err := tx.Exec(sqlStatement)
return err
}

8
server/datastore/mysql/migrations/tables/20170306075207_UseUTF8MB.go
View file

@ -3,6 +3,7 @@ package tables
import (
"database/sql"
"fmt"
"os"
"github.com/pkg/errors"
)
@ -14,6 +15,11 @@ func init() {
// changeCharacterSet changes the default character set of the database and all
// table to the provided character set
func changeCharacterSet(tx *sql.Tx, charset string) error {
// This env var should only be set during TestCollation.
if v := os.Getenv("FLEET_TEST_DISABLE_COLLATION_UPDATES"); v != "" {
return nil
}
_, err := tx.Exec(fmt.Sprintf("ALTER DATABASE DEFAULT CHARACTER SET %s", charset))
if err != nil {
return errors.Wrap(err, "alter database")
@ -45,7 +51,7 @@ func changeCharacterSet(tx *sql.Tx, charset string) error {
rows.Close()
for _, name := range names {
_, err = tx.Exec(fmt.Sprintf("ALTER TABLE %s CONVERT TO CHARACTER SET %s", name, charset))
_, err = tx.Exec(fmt.Sprintf("ALTER TABLE %s CONVERT TO CHARACTER SET %s COLLATE utf8mb4_unicode_ci", name, charset))
if err != nil {
return errors.Wrap(err, "alter table "+name)
}

2
server/datastore/mysql/migrations/tables/20171116163618_CreateTableOsqueryOptions.go
View file

@ -21,7 +21,7 @@ func Up_20171116163618(tx *sql.Tx) error {
"`override_identifier` VARCHAR(255) NOT NULL DEFAULT ''," +
"`options` JSON NOT NULL," +
"PRIMARY KEY (`id`)" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8;"
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"
_, err := tx.Exec(sqlStatement)
if err != nil {
return errors.Wrap(err, "create table osquery_options")

4
server/datastore/mysql/migrations/tables/20171219164727_UpdateScheduledQueryKey.go
View file

@ -60,8 +60,8 @@ func Up_20171219164727(tx *sql.Tx) error {
query = `
ALTER TABLE scheduled_queries
ADD COLUMN name varchar(255),
ADD COLUMN description varchar(255)
ADD COLUMN name varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci,
ADD COLUMN description varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci
`
if _, err := tx.Exec(query); err != nil {
return errors.Wrap(err, "adding name to scheduled_queries")

4
server/datastore/mysql/migrations/tables/20200311140000_AddHostMacAndIp.go
View file

@ -11,8 +11,8 @@ func init() {
func Up20200311140000(tx *sql.Tx) error {
_, err := tx.Exec(
"ALTER TABLE `hosts` " +
"ADD COLUMN `primary_ip` varchar(45) NOT NULL DEFAULT ''," +
"ADD COLUMN `primary_mac` varchar(17) NOT NULL DEFAULT ''," +
"ADD COLUMN `primary_ip` varchar(45) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''," +
"ADD COLUMN `primary_mac` varchar(17) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''," +
"ADD FULLTEXT KEY `host_ip_mac_search` (`primary_ip`,`primary_mac`)",
)
return err

4
server/datastore/mysql/migrations/tables/20200512120000_CreateTableEnrollSecrets.go
View file

@ -18,7 +18,7 @@ func Up_20200512120000(tx *sql.Tx) error {
"`secret` VARCHAR(255) NOT NULL," +
"`active` TINYINT(1) DEFAULT TRUE," +
"PRIMARY KEY (`name`)" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;",
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;",
)
if err != nil {
return errors.Wrap(err, "create enroll_secrets table")
@ -34,7 +34,7 @@ func Up_20200512120000(tx *sql.Tx) error {
_, err = tx.Exec(
"ALTER TABLE `hosts`" +
"ADD COLUMN `enroll_secret_name` VARCHAR(255) NOT NULL DEFAULT ''",
"ADD COLUMN `enroll_secret_name` VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''",
)
if err != nil {
return errors.Wrap(err, "drop old secret column")

2
server/datastore/mysql/migrations/tables/20201021104586_CreateCarveTables.go
View file

@ -27,7 +27,7 @@ func Up_20201021104586(tx *sql.Tx) error {
UNIQUE KEY idx_session_id (session_id),
UNIQUE KEY idx_name (name),
FOREIGN KEY (host_id) REFERENCES hosts (id) ON DELETE CASCADE
)`); err != nil {
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`); err != nil {
return errors.Wrap(err, "create carve_metadata")
}

2
server/datastore/mysql/migrations/tables/20210421112652_AddSoftwareInventory.go
View file

@ -19,7 +19,7 @@ func Up_20210421112652(tx *sql.Tx) error {
version varchar(255) NOT NULL DEFAULT '',
source varchar(64) NOT NULL,
UNIQUE KEY idx_name_version (name, version, source)
)`); err != nil {
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`); err != nil {
return errors.Wrap(err, "create table software")
}

6
server/datastore/mysql/migrations/tables/20210601000001_CreateTeamsTables.go
View file

@ -17,7 +17,7 @@ func Up_20210601000001(tx *sql.Tx) error {
name VARCHAR(255) NOT NULL,
description VARCHAR(1023) NOT NULL DEFAULT '',
UNIQUE KEY idx_name (name)
)`); err != nil {
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`); err != nil {
return errors.Wrap(err, "create teams")
}
@ -29,7 +29,7 @@ func Up_20210601000001(tx *sql.Tx) error {
PRIMARY KEY (user_id, team_id),
FOREIGN KEY fk_user_teams_user_id (user_id) REFERENCES users (id) ON DELETE CASCADE ON UPDATE CASCADE,
FOREIGN KEY fk_user_teams_team_id (team_id) REFERENCES teams (id) ON DELETE CASCADE ON UPDATE CASCADE
)`); err != nil {
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`); err != nil {
return errors.Wrap(err, "create user_teams")
}
@ -41,7 +41,7 @@ func Up_20210601000001(tx *sql.Tx) error {
}
if _, err := tx.Exec(`ALTER TABLE users
ADD global_role VARCHAR(64) DEFAULT NULL
ADD global_role VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL
`); err != nil {
return errors.Wrap(err, "alter users")
}

2
server/datastore/mysql/migrations/tables/20210601000002_AddRolesToInvites.go
View file

@ -19,7 +19,7 @@ func Up_20210601000002(tx *sql.Tx) error {
PRIMARY KEY (invite_id, team_id),
FOREIGN KEY fk_invite_id (invite_id) REFERENCES invites (id) ON DELETE CASCADE ON UPDATE CASCADE,
FOREIGN KEY fk_team_id (team_id) REFERENCES teams (id) ON DELETE CASCADE ON UPDATE CASCADE
)`); err != nil {
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`); err != nil {
return errors.Wrap(err, "create invite_teams")
}

2
server/datastore/mysql/migrations/tables/20210708143152_AddHostUsers.go
View file

@ -21,7 +21,7 @@ func Up_20210708143152(tx *sql.Tx) error {
user_type VARCHAR(255),
UNIQUE KEY idx_uid_username (host_id, uid, username),
FOREIGN KEY (host_id) REFERENCES hosts (id) ON DELETE CASCADE
)`
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`
if _, err := tx.Exec(sqlStatement); err != nil {
return err
}

3
server/datastore/mysql/migrations/tables/20210709124443_CreateActivitiesTable.go
View file

@ -2,6 +2,7 @@ package tables
import (
"database/sql"
"github.com/pkg/errors"
)
@ -20,7 +21,7 @@ func Up_20210709124443(tx *sql.Tx) error {
details json DEFAULT NULL,
PRIMARY KEY (id),
FOREIGN KEY fk_activities_user_id (user_id) REFERENCES users (id) ON DELETE SET NULL
)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`
if _, err := tx.Exec(sql); err != nil {
return errors.Wrap(err, "create activities")

3
server/datastore/mysql/migrations/tables/20210712155608_CreateLocksTable.go
View file

@ -2,6 +2,7 @@ package tables
import (
"database/sql"
"github.com/pkg/errors"
)
@ -17,7 +18,7 @@ func Up_20210712155608(tx *sql.Tx) error {
owner VARCHAR(255),
expires_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
UNIQUE KEY idx_name (name)
)`); err != nil {
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`); err != nil {
return errors.Wrap(err, "create locks")
}
return nil

2
server/datastore/mysql/migrations/tables/20210719153709_AddStatisticsTable.go
View file

@ -18,7 +18,7 @@ func Up_20210719153709(tx *sql.Tx) error {
updated_at timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
anonymous_identifier varchar(255) NOT NULL,
PRIMARY KEY (id)
)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`
if _, err := tx.Exec(sql); err != nil {
return errors.Wrap(err, "create statistics")

2
server/datastore/mysql/migrations/tables/20210721171531_AddCPETable.go
View file

@ -20,7 +20,7 @@ func Up_20210721171531(tx *sql.Tx) error {
cpe varchar(255) NOT NULL,
PRIMARY KEY (id),
FOREIGN KEY fk_software_cpe_software_id (software_id) REFERENCES software(id) ON DELETE CASCADE
)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`
if _, err := tx.Exec(sql); err != nil {
return errors.Wrap(err, "create cpe")

2
server/datastore/mysql/migrations/tables/20210802135933_AddCVETable.go
View file

@ -21,7 +21,7 @@ func Up_20210802135933(tx *sql.Tx) error {
PRIMARY KEY (id),
FOREIGN KEY fk_software_cve_cpe_id (cpe_id) REFERENCES software_cpe(id) ON DELETE CASCADE,
UNIQUE KEY unique_cpe_cve(cpe_id, cve)
)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`
if _, err := tx.Exec(sql); err != nil {
return errors.Wrap(err, "create cve")

2
server/datastore/mysql/migrations/tables/20210819143446_AddPolicies.go
View file

@ -19,7 +19,7 @@ func Up_20210819143446(tx *sql.Tx) error {
updated_at timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id),
FOREIGN KEY fk_policies_query_id (query_id) REFERENCES queries(id) ON DELETE RESTRICT
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`
policyMembershipHistoryTable := `
CREATE TABLE IF NOT EXISTS policy_membership_history (

2
server/datastore/mysql/migrations/tables/20211013133707_AddAggregatedStats.go
View file

@ -20,7 +20,7 @@ func Up_20211013133707(tx *sql.Tx) error {
updated_at timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id, type),
INDEX idx_aggregated_stats_updated_at(updated_at)
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`
if _, err := tx.Exec(aggregatedStatsTable); err != nil {
return errors.Wrap(err, "create aggregated stats table")

4
server/datastore/mysql/migrations/tables/20211216131203_AddMDMMunkiTables.go
View file

@ -18,7 +18,7 @@ func Up_20211216131203(tx *sql.Tx) error {
server_url VARCHAR(255) DEFAULT '' NOT NULL,
installed_from_dep bool DEFAULT FALSE NOT NULL,
PRIMARY KEY (host_id)
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`
if _, err := tx.Exec(mdmTable); err != nil {
return errors.Wrap(err, "create host_mdm table")
@ -28,7 +28,7 @@ func Up_20211216131203(tx *sql.Tx) error {
host_id int(10) UNSIGNED NOT NULL,
version VARCHAR(255) DEFAULT '' NOT NULL,
PRIMARY KEY (host_id)
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`
if _, err := tx.Exec(munkiInfoTable); err != nil {
return errors.Wrap(err, "create host_munki_info table")

2
server/datastore/mysql/migrations/tables/20211221110132_AddHostEmailsTable.go
View file

@ -23,7 +23,7 @@ func Up_20211221110132(tx *sql.Tx) error {
PRIMARY KEY (id),
INDEX idx_host_emails_host_id_email (host_id, email),
INDEX idx_host_emails_email (email)
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`
if _, err := tx.Exec(hostEmailsTable); err != nil {
return errors.Wrap(err, "create host_emails table")

2
server/datastore/mysql/migrations/tables/20220307104655_AddHostDeviceAuth.go
View file

@ -17,7 +17,7 @@ func Up_20220307104655(tx *sql.Tx) error {
token VARCHAR(255) NOT NULL,
PRIMARY KEY (host_id),
UNIQUE INDEX idx_host_device_auth_token (token)
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`
if _, err := tx.Exec(hostDeviceAuthTable); err != nil {
return errors.Wrap(err, "create host_device_auth table")

3
server/datastore/mysql/migrations/tables/20220316155700_AddPublicIPHosts.go
View file

@ -12,7 +12,7 @@ func init() {
func Up_20220316155700(tx *sql.Tx) error {
_, err := tx.Exec(
"ALTER TABLE `hosts` ADD COLUMN `public_ip` varchar(45) NOT NULL DEFAULT ''",
"ALTER TABLE `hosts` ADD COLUMN `public_ip` varchar(45) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''",
)
if err != nil {
return errors.Wrap(err, "add public_ip column")
@ -20,6 +20,7 @@ func Up_20220316155700(tx *sql.Tx) error {
return nil
}
func Down_20220316155700(tx *sql.Tx) error {
return nil
}

2
server/datastore/mysql/migrations/tables/20220330100659_AddJobsTable.go
View file

@ -21,7 +21,7 @@ CREATE TABLE jobs (
state VARCHAR(255) NOT NULL,
retries INT NOT NULL DEFAULT 0,
error TEXT
)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`)
if err != nil {
return errors.Wrapf(err, "create table")

2
server/datastore/mysql/migrations/tables/20220503134048_AddCVEScoresTable.go
View file

@ -17,7 +17,7 @@ CREATE TABLE cve_scores (
cvss_score double,
epss_probability double,
cisa_known_exploit boolean
)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`)
if err != nil {
return errors.Wrapf(err, "create table")

2
server/datastore/mysql/migrations/tables/20220627104817_AddHostBatteriesTable.go
View file

@ -24,7 +24,7 @@ CREATE TABLE host_batteries (
updated_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY idx_host_batteries_host_id_serial_number (host_id, serial_number)
)`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
if err != nil {
return errors.Wrapf(err, "create table")
}

2
server/datastore/mysql/migrations/tables/20220713091130_AddOperatingSystemsTable.go
View file

@ -21,7 +21,7 @@ CREATE TABLE operating_systems (
kernel_version VARCHAR(150) NOT NULL,
platform VARCHAR(50) NOT NULL,
UNIQUE KEY idx_unique_os (name, version, arch, kernel_version, platform)
)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`)
if err != nil {
return errors.Wrapf(err, "create operating_systems table")

2
server/datastore/mysql/migrations/tables/20220802135510_AddMobileDeviceManagementTable.go
View file

@ -22,7 +22,7 @@ CREATE TABLE mobile_device_management_solutions (
updated_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY idx_mobile_device_management_solutions_name (name, server_url)
)`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
if err != nil {
return errors.Wrapf(err, "create table")
}

2
server/datastore/mysql/migrations/tables/20220822161445_CreateMunkiIssuesTable.go
View file

@ -28,7 +28,7 @@ func Up_20220822161445(tx *sql.Tx) error {
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE KEY idx_munki_issues_name (name, issue_type)
)`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
if err != nil {
return errors.Wrapf(err, "create munki_issues table")
}

2
server/datastore/mysql/migrations/tables/20220908181826_AddOrbitNodeKeyToHosts_test.go
View file

@ -26,8 +26,8 @@ func TestUp_20220908181826(t *testing.T) {
require.NoError(t, err)
applyNext(t, db)
sqlUpdate := `UPDATE hosts SET orbit_node_key = ? WHERE osquery_host_id = ?`
_, err = db.Exec(sqlUpdate, "orbit_node_key", "host_id")
require.NoError(t, err)
}

4
server/datastore/mysql/migrations/tables/20220915165115_AppleMDMTables.go
View file

@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS mdm_apple_enrollment_profiles (
PRIMARY KEY (id),
UNIQUE KEY idx_token (token),
UNIQUE KEY idx_type (type)
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`)
if err != nil {
return fmt.Errorf("failed to create mdm_apple_enrollments table: %w", err)
@ -68,7 +68,7 @@ CREATE TABLE IF NOT EXISTS mdm_apple_installers (
url_token VARCHAR(36) DEFAULT NULL,
PRIMARY KEY (id)
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`)
if err != nil {
return fmt.Errorf("failed to create mdm_apple_installers table: %w", err)

2
server/datastore/mysql/migrations/tables/20220915165115_AppleMDMTables_nanodep.sql
View file

@ -33,4 +33,4 @@ CREATE TABLE nano_dep_names (
CHECK (tokenpki_cert_pem IS NULL OR SUBSTRING(tokenpki_cert_pem FROM 1 FOR 27) = '-----BEGIN CERTIFICATE-----'),
CHECK (tokenpki_key_pem IS NULL OR SUBSTRING(tokenpki_key_pem FROM 1 FOR 5) = '-----')
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

20
server/datastore/mysql/migrations/tables/20220915165115_AppleMDMTables_nanomdm.sql
View file

@ -3,7 +3,7 @@
*/
CREATE TABLE nano_devices (
id VARCHAR(255) NOT NULL,
id VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
identity_cert TEXT NULL,
@ -41,7 +41,7 @@ CREATE TABLE nano_devices (
CHECK (token_update IS NULL OR token_update != ''),
CHECK (bootstrap_token_b64 IS NULL OR bootstrap_token_b64 != '')
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE nano_users (
@ -77,7 +77,7 @@ CREATE TABLE nano_users (
CHECK (user_authenticate IS NULL OR user_authenticate != ''),
CHECK (user_authenticate_digest IS NULL OR user_authenticate_digest != '')
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
ALTER TABLE nano_users ADD CONSTRAINT idx_unique_id UNIQUE (id);
@ -86,7 +86,7 @@ ALTER TABLE nano_users ADD CONSTRAINT idx_unique_id UNIQUE (id);
*/
CREATE TABLE nano_enrollments (
-- The enrollment ID of this enrollment
id VARCHAR(255) NOT NULL,
id VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
-- The "device" enrollment ID of this enrollment. This will be
-- the same as the `id` field in the case of a "device" enrollment,
-- or will be the "parent" enrollment for a "user" enrollment.
@ -130,7 +130,7 @@ CREATE TABLE nano_enrollments (
CHECK (topic != ''),
CHECK (push_magic != ''),
CHECK (token_hex != '')
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/* Commands stand alone. By themsevles they aren't associated with
@ -151,7 +151,7 @@ CREATE TABLE nano_commands (
CHECK (command_uuid != ''),
CHECK (request_type != ''),
CHECK (SUBSTRING(command FROM 1 FOR 5) = '<?xml')
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/* Results are enrollment responses to device commands.
@ -192,7 +192,7 @@ CREATE TABLE nano_command_results (
CHECK (status != ''),
INDEX (status),
CHECK (SUBSTRING(result FROM 1 FOR 5) = '<?xml')
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE nano_enrollment_queue (
@ -214,7 +214,7 @@ CREATE TABLE nano_enrollment_queue (
FOREIGN KEY (command_uuid)
REFERENCES nano_commands (command_uuid)
ON DELETE CASCADE ON UPDATE CASCADE
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/* An enrollment's queue is a view into commands, enrollment queued
* commands, and any results received. Outstanding queue items (i.e.
@ -268,7 +268,7 @@ CREATE TABLE nano_push_certs (
CHECK (SUBSTRING(cert_pem FROM 1 FOR 27) = '-----BEGIN CERTIFICATE-----'),
CHECK (SUBSTRING(key_pem FROM 1 FOR 5) = '-----')
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE nano_cert_auth_associations (
@ -282,4 +282,4 @@ CREATE TABLE nano_cert_auth_associations (
CHECK (id != ''),
CHECK (sha256 != '')
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

2
server/datastore/mysql/migrations/tables/20220915165115_AppleMDMTables_scep.sql
View file

@ -37,4 +37,4 @@ CREATE TABLE IF NOT EXISTS scep_certificates (
CHECK (SUBSTRING(certificate_pem FROM 1 FOR 27) = '-----BEGIN CERTIFICATE-----'),
CHECK (name IS NULL OR name != '')
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

4
server/datastore/mysql/migrations/tables/20220915165116_HostDisplayName.go
View file

@ -2,6 +2,7 @@ package tables
import (
"database/sql"
"github.com/pkg/errors"
)
@ -19,7 +20,7 @@ func Up_20220915165116(tx *sql.Tx) error {
display_name varchar(255) NOT NULL,
PRIMARY KEY (host_id),
KEY (display_name)
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`},
{"migrate data", `
INSERT INTO host_display_names (
@ -30,7 +31,6 @@ func Up_20220915165116(tx *sql.Tx) error {
if _, err := tx.Exec(change.sql); err != nil {
return errors.Wrapf(err, "upHostDisplayName: %s", change.name)
}
}
return nil
}

2
server/datastore/mysql/migrations/tables/20221014084130_CreateHostOrbitInfoTable.go
View file

@ -16,7 +16,7 @@ func Up_20221014084130(tx *sql.Tx) error {
PRIMARY KEY (host_id),
KEY idx_host_orbit_info_version (version)
)`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
return err
}

2
server/datastore/mysql/migrations/tables/20221027085019_CreateOperatingSystemVulnerabilitiesTable.go
View file

@ -25,7 +25,7 @@ func Up_20221027085019(tx *sql.Tx) error {
UNIQUE KEY idx_operating_system_vulnerabilities_unq_cve (host_id, cve),
INDEX idx_operating_system_vulnerabilities_operating_system_id_cve (operating_system_id, cve)
)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`)
if err != nil {
return errors.Wrapf(err, "operating_system_vulnerabilities")

4
server/datastore/mysql/migrations/tables/20221115104546_AddCronStatsTable.go
View file

@ -20,7 +20,9 @@ func Up_20221115104546(tx *sql.Tx) error {
status VARCHAR(255) NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
INDEX idx_cron_stats_name_created_at (name, created_at))
INDEX idx_cron_stats_name_created_at (name, created_at)
)
ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`)
if err != nil {
return errors.Wrapf(err, "create table")

2
server/datastore/mysql/migrations/tables/20221223174807_AlterHostsTablePendingMDMEnrollments.go
View file

@ -12,7 +12,7 @@ func init() {
func Up_20221223174807(tx *sql.Tx) error {
_, err := tx.Exec(`
ALTER TABLE hosts MODIFY osquery_host_id VARCHAR(255) NULL;
ALTER TABLE hosts MODIFY osquery_host_id VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL;
ALTER TABLE hosts ADD INDEX idx_hosts_hardware_serial (hardware_serial)
`)
if err != nil {

2
server/datastore/mysql/migrations/tables/20230202224725_CreateHostDiskEncryptionKeysTable.go
View file

@ -24,7 +24,7 @@ func Up_20230202224725(tx *sql.Tx) error {
PRIMARY KEY (host_id),
KEY idx_host_disk_encryption_keys_decryptable (decryptable)
)`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
return err
}

2
server/datastore/mysql/migrations/tables/20230206163608_AddTableMDMAppleConfigProfiles.go
View file

@ -25,7 +25,7 @@ CREATE TABLE mdm_apple_configuration_profiles (
PRIMARY KEY (profile_id),
UNIQUE KEY idx_mdm_apple_config_prof_team_identifier (team_id, identifier),
UNIQUE KEY idx_mdm_apple_config_prof_team_name (team_id, name)
);`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
if err != nil {
return errors.Wrapf(err, "create table")
}

6
server/datastore/mysql/migrations/tables/20230214131519_AddHostMDMAppleProfilesTable.go
View file

@ -12,7 +12,7 @@ func Up_20230214131519(tx *sql.Tx) error {
_, err := tx.Exec(`
CREATE TABLE mdm_apple_delivery_status (
status VARCHAR(20) PRIMARY KEY
)`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`)
if err != nil {
return err
}
@ -28,7 +28,7 @@ func Up_20230214131519(tx *sql.Tx) error {
_, err = tx.Exec(`
CREATE TABLE mdm_apple_operation_types (
operation_type VARCHAR(20) PRIMARY KEY
)`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`)
if err != nil {
return err
}
@ -54,7 +54,7 @@ func Up_20230214131519(tx *sql.Tx) error {
PRIMARY KEY (host_uuid, profile_id),
FOREIGN KEY (status) REFERENCES mdm_apple_delivery_status (status) ON UPDATE CASCADE,
FOREIGN KEY (operation_type) REFERENCES mdm_apple_operation_types (operation_type) ON UPDATE CASCADE
)`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
return err
}

2
server/datastore/mysql/migrations/tables/20230303135738_AddMDMIdPAccountsTable.go
View file

@ -18,7 +18,7 @@ func Up_20230303135738(tx *sql.Tx) error {
iterations int unsigned NOT NULL,
PRIMARY KEY (uuid)
)`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
return err
}

2
server/datastore/mysql/migrations/tables/20230313135301_AddProfileNameColumnToHostMDMAppleProfilesTable.go
View file

@ -11,7 +11,7 @@ func init() {
}
func Up_20230313135301(tx *sql.Tx) error {
if _, err := tx.Exec("ALTER TABLE `host_mdm_apple_profiles` ADD COLUMN `profile_name` varchar(255) NOT NULL DEFAULT ''"); err != nil {
if _, err := tx.Exec("ALTER TABLE `host_mdm_apple_profiles` ADD COLUMN `profile_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''"); err != nil {
return errors.Wrap(err, "adding profile_name column to `host_mdm_apple_profiles")
}
return nil

17
server/datastore/mysql/migrations/tables/20230315104937_EnsureUniformCollation.go
View file

@ -5,6 +5,7 @@ import (
"database/sql"
"encoding/json"
"fmt"
"os"
"text/template"
"github.com/jmoiron/sqlx"
@ -36,7 +37,6 @@ func fixupSoftware(tx *sql.Tx, collation string) error {
arch COLLATE %s
HAVING total > 1
COLLATE %s`, collation, collation, collation, collation, collation))
if err != nil {
return fmt.Errorf("aggregating dupes: %w", err)
}
@ -208,6 +208,11 @@ func fixupOS(tx *sql.Tx, collation string) error {
// This is based on the changeCharacterSet function that's included in this
// module and part of the 20170306075207_UseUTF8MB migration.
func changeCollation(tx *sql.Tx, charset string, collation string) (err error) {
// This env var should only be set during TestCollation.
if v := os.Getenv("FLEET_TEST_DISABLE_COLLATION_UPDATES"); v != "" {
return nil
}
_, err = tx.Exec(fmt.Sprintf("ALTER DATABASE DEFAULT CHARACTER SET `%s` COLLATE `%s`", charset, collation))
if err != nil {
return fmt.Errorf("alter database: %w", err)
@ -228,11 +233,11 @@ func changeCollation(tx *sql.Tx, charset string, collation string) (err error) {
txx := sqlx.Tx{Tx: tx}
var names []string
err = txx.Select(&names, `
SELECT table_name
FROM information_schema.TABLES AS T, information_schema.COLLATION_CHARACTER_SET_APPLICABILITY AS C
WHERE C.collation_name = T.table_collation
AND T.table_schema = (SELECT database())
AND (C.CHARACTER_SET_NAME != ? OR C.COLLATION_NAME != ?)
SELECT table_name
FROM information_schema.TABLES AS T, information_schema.COLLATION_CHARACTER_SET_APPLICABILITY AS C
WHERE C.collation_name = T.table_collation
AND T.table_schema = (SELECT database())
AND (C.CHARACTER_SET_NAME != ? OR C.COLLATION_NAME != ?)
-- exclude tables that have columns with specific collations
AND table_name NOT IN ('hosts', 'enroll_secrets')`, charset, collation)
if err != nil {

4
server/datastore/mysql/migrations/tables/20230411102858_CreateHostBootstrapPackagesTable.go
View file

@ -13,8 +13,8 @@ func Up_20230411102858(tx *sql.Tx) error {
// create host_mdm_apple_bootstrap_packages table
_, err := tx.Exec(`
CREATE TABLE host_mdm_apple_bootstrap_packages (
host_uuid varchar(127) NOT NULL,
command_uuid varchar(127) NOT NULL,
host_uuid varchar(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
command_uuid varchar(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
PRIMARY KEY (host_uuid),
FOREIGN KEY (command_uuid) REFERENCES nano_commands (command_uuid) ON DELETE CASCADE

2
server/datastore/mysql/migrations/tables/20230518114155_AddFullnameToMDMIdPAccountsTable.go
View file

@ -15,7 +15,7 @@ func Up_20230518114155(tx *sql.Tx) error {
DROP COLUMN salt,
DROP COLUMN entropy,
DROP COLUMN iterations,
ADD COLUMN fullname varchar(256) NOT NULL DEFAULT ''
ADD COLUMN fullname varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''
`
if _, err := tx.Exec(stmt); err != nil {
return fmt.Errorf("alter mdm_idp_accounts table: %w", err)

6
server/datastore/mysql/migrations/tables/20230721135421_QueriesSchemaChanges.go
View file

@ -29,12 +29,12 @@ func Up_20230721135421(tx *sql.Tx) error {
ADD team_id INT(10) UNSIGNED DEFAULT NULL,
ADD team_id_char CHAR(10) DEFAULT '' NOT NULL,
ADD platform VARCHAR(255) DEFAULT '' NOT NULL,
ADD min_osquery_version VARCHAR(255) DEFAULT '' NOT NULL,
ADD platform VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT '' NOT NULL,
ADD min_osquery_version VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT '' NOT NULL,
ADD schedule_interval INT(10) UNSIGNED DEFAULT 0 NOT NULL,
ADD automations_enabled TINYINT(1) UNSIGNED DEFAULT 0 NOT NULL,
ADD logging_type VARCHAR(255) DEFAULT 'snapshot' NOT NULL,
ADD logging_type VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT 'snapshot' NOT NULL,
ADD FOREIGN KEY fk_queries_team_id (team_id) REFERENCES teams (id) ON DELETE CASCADE,
ADD UNIQUE INDEX idx_team_id_name_unq (team_id_char, name);

16
server/datastore/mysql/migrations/tables/20250127162751_AddUnifiedQueueTable.go
View file

@ -3,6 +3,7 @@ package tables
import (
"database/sql"
"fmt"
"os"
"github.com/jmoiron/sqlx"
)
@ -17,6 +18,11 @@ func init() {
// This is based on the changeCollation function that's included in this
// module and part of the 20230315104937_EnsureUniformCollation migration.
func changeCollation2025(tx *sql.Tx, charset string, collation string) (err error) {
// This env var should only be set during TestCollation.
if v := os.Getenv("FLEET_TEST_DISABLE_COLLATION_UPDATES"); v != "" {
return nil
}
_, err = tx.Exec(fmt.Sprintf("ALTER DATABASE DEFAULT CHARACTER SET `%s` COLLATE `%s`", charset, collation))
if err != nil {
return fmt.Errorf("alter database: %w", err)
@ -25,11 +31,11 @@ func changeCollation2025(tx *sql.Tx, charset string, collation string) (err erro
txx := sqlx.Tx{Tx: tx}
var names []string
err = txx.Select(&names, `
SELECT table_name
FROM information_schema.TABLES AS T, information_schema.COLLATION_CHARACTER_SET_APPLICABILITY AS C
WHERE C.collation_name = T.table_collation
AND T.table_schema = (SELECT database())
AND (C.CHARACTER_SET_NAME != ? OR C.COLLATION_NAME != ?)
SELECT table_name
FROM information_schema.TABLES AS T, information_schema.COLLATION_CHARACTER_SET_APPLICABILITY AS C
WHERE C.collation_name = T.table_collation
AND T.table_schema = (SELECT database())
AND (C.CHARACTER_SET_NAME != ? OR C.COLLATION_NAME != ?)
-- exclude tables that have columns with specific collations
AND table_name NOT IN ('hosts', 'enroll_secrets')`, charset, collation)
if err != nil {

4
server/datastore/mysql/migrations/tables/20250501162727_AddSoftwareCategories.go
View file

@ -12,9 +12,9 @@ func init() {
func Up_20250501162727(tx *sql.Tx) error {
_, err := tx.Exec(`CREATE TABLE IF NOT EXISTS software_categories (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(63) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
name VARCHAR(63) NOT NULL,
UNIQUE KEY idx_software_categories_name (name)
)`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
if err != nil {
return fmt.Errorf("failed to create software_categories table: %w", err)
}

33
server/datastore/mysql/migrations/tables/20250529102706_SetMediumTextColumns.go Normal file
View file

@ -0,0 +1,33 @@
package tables
import (
"database/sql"
"fmt"
)
func init() {
MigrationClient.AddMigration(Up_20250529102706, Down_20250529102706)
}
func Up_20250529102706(tx *sql.Tx) error {
_, err := tx.Exec(
`ALTER TABLE queries
MODIFY query MEDIUMTEXT NOT NULL,
MODIFY description MEDIUMTEXT NOT NULL;
`,
)
if err != nil {
return fmt.Errorf("failed to set queries.query and queries.description to mediumtext: %w", err)
}
_, err = tx.Exec(
`ALTER TABLE labels MODIFY query MEDIUMTEXT NOT NULL;`,
)
if err != nil {
return fmt.Errorf("failed to set labels.query to mediumtext: %w", err)
}
return nil
}
func Down_20250529102706(tx *sql.Tx) error {
return nil
}

31
server/datastore/mysql/migrations/tables/collation_test.go Normal file
View file

@ -0,0 +1,31 @@
package tables
import (
"errors"
"os"
"testing"
"github.com/fleetdm/fleet/v4/server/goose"
"github.com/stretchr/testify/require"
)
func TestCollation(t *testing.T) {
require.NoError(t, os.Setenv("FLEET_TEST_DISABLE_COLLATION_UPDATES", "true"))
t.Cleanup(func() {
require.NoError(t, os.Unsetenv("FLEET_TEST_DISABLE_COLLATION_UPDATES"))
})
db := newDBConnForTests(t)
for {
current, err := MigrationClient.GetDBVersion(db.DB)
require.NoError(t, err)
_, err = MigrationClient.Migrations.Next(current)
if errors.Is(err, goose.ErrNoNextVersion) {
break
}
require.NoError(t, err)
applyNext(t, db)
}
checkCollation(t, db)
}

30
server/datastore/mysql/migrations/tables/migration_test.go
View file

@ -155,6 +155,36 @@ func insertQuery(t *testing.T, db *sqlx.DB) uint {
return uint(id) //nolint:gosec // dismiss G115
}
func checkCollation(t *testing.T, db *sqlx.DB) {
type collationData struct {
CollationName string `db:"COLLATION_NAME"`
TableName string `db:"TABLE_NAME"`
ColumnName string `db:"COLUMN_NAME"`
CharacterSetName string `db:"CHARACTER_SET_NAME"`
}
stmt := `
SELECT
TABLE_NAME, COLLATION_NAME, COLUMN_NAME, CHARACTER_SET_NAME
FROM information_schema.columns
WHERE
TABLE_SCHEMA = (SELECT DATABASE())
AND (CHARACTER_SET_NAME != ? OR COLLATION_NAME != ?)`
var nonStandardCollations []collationData
err := db.Select(&nonStandardCollations, stmt, "utf8mb4", "utf8mb4_unicode_ci")
require.NoError(t, err)
exceptions := []collationData{
{"utf8mb4_bin", "enroll_secrets", "secret", "utf8mb4"},
{"utf8mb4_bin", "hosts", "node_key", "utf8mb4"},
{"utf8mb4_bin", "hosts", "orbit_node_key", "utf8mb4"},
{"utf8mb4_bin", "teams", "name_bin", "utf8mb4"},
}
require.ElementsMatch(t, exceptions, nonStandardCollations)
}
func insertHost(t *testing.T, db *sqlx.DB, teamID *uint) uint {
// Insert a minimal record into hosts table
insertHostStmt := `

62
server/datastore/mysql/schema.sql
View file

File diff suppressed because one or more lines are too long