Merge Android datastore into main Fleet datastore (#32233)

Resolves #31218
This commit is contained in:
Carlo 2025-08-25 11:41:28 -04:00 committed by GitHub
parent 2fd6a86f41
commit 8bc8d01f0a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 167 additions and 322 deletions

View file

@ -10,7 +10,6 @@ on:
paths: paths:
- '**.go' - '**.go'
- 'server/datastore/mysql/schema.sql' - 'server/datastore/mysql/schema.sql'
- 'server/mdm/android/mysql/schema.sql'
- '.github/workflows/test-db-changes.yml' - '.github/workflows/test-db-changes.yml'
workflow_dispatch: # Manual workflow_dispatch: # Manual
@ -93,7 +92,7 @@ jobs:
- name: Verify test schema changes - name: Verify test schema changes
run: | run: |
make test-schema make test-schema
if [[ $(git diff-files --patch server/datastore/mysql/schema.sql server/mdm/android/mysql/schema.sql) ]]; then if [[ $(git diff-files --patch server/datastore/mysql/schema.sql) ]]; then
echo "❌ fail: uncommited changes in schema.sql" echo "❌ fail: uncommited changes in schema.sql"
echo "please run 'make test-schema' and commit the changes" echo "please run 'make test-schema' and commit the changes"
exit 1 exit 1

View file

@ -232,7 +232,7 @@ endif
.help-short--test-schema: .help-short--test-schema:
@echo "Update schema.sql from current migrations" @echo "Update schema.sql from current migrations"
test-schema: test-schema:
go run ./tools/dbutils ./server/datastore/mysql/schema.sql ./server/mdm/android/mysql/schema.sql go run ./tools/dbutils ./server/datastore/mysql/schema.sql
dump-test-schema: test-schema dump-test-schema: test-schema
# This is the base command to run Go tests. # This is the base command to run Go tests.
@ -307,7 +307,7 @@ FAST_PKGS_TO_TEST := \
./server/mdm/scep/x509util \ ./server/mdm/scep/x509util \
./server/policies ./server/policies
FLEETCTL_PKGS_TO_TEST := ./cmd/fleetctl/... FLEETCTL_PKGS_TO_TEST := ./cmd/fleetctl/...
MYSQL_PKGS_TO_TEST := ./server/datastore/mysql/... ./server/mdm/android/mysql MYSQL_PKGS_TO_TEST := ./server/datastore/mysql/...
SCRIPTS_PKGS_TO_TEST := ./orbit/pkg/scripts SCRIPTS_PKGS_TO_TEST := ./orbit/pkg/scripts
SERVICE_PKGS_TO_TEST := ./server/service SERVICE_PKGS_TO_TEST := ./server/service
VULN_PKGS_TO_TEST := ./server/vulnerabilities/... VULN_PKGS_TO_TEST := ./server/vulnerabilities/...

View file

@ -3,7 +3,7 @@
## DB Schema Dump ## DB Schema Dump
if [[ $DB_SCHEMA_DUMP ]]; then if [[ $DB_SCHEMA_DUMP ]]; then
make dump-test-schema make dump-test-schema
if [[ $(git diff-files --patch server/datastore/mysql/schema.sql server/mdm/android/mysql/schema.sql) ]]; then if [[ $(git diff-files --patch server/datastore/mysql/schema.sql) ]]; then
echo "❌ fail: uncommited changes in schema.sql" echo "❌ fail: uncommited changes in schema.sql"
echo "please run 'make dump-test-schema' and commit the changes" echo "please run 'make dump-test-schema' and commit the changes"
exit 1 exit 1

View file

@ -116,7 +116,7 @@ func (ds *Datastore) NewAndroidHost(ctx context.Context, host *fleet.AndroidHost
return ctxerr.Wrap(ctx, err, "new Android host MDM info") return ctxerr.Wrap(ctx, err, "new Android host MDM info")
} }
host.Device, err = ds.Datastore.CreateDeviceTx(ctx, tx, host.Device) host.Device, err = ds.CreateDeviceTx(ctx, tx, host.Device)
if err != nil { if err != nil {
return ctxerr.Wrap(ctx, err, "creating new Android device") return ctxerr.Wrap(ctx, err, "creating new Android device")
} }
@ -203,7 +203,7 @@ func (ds *Datastore) UpdateAndroidHost(ctx context.Context, host *fleet.AndroidH
} }
} }
err = ds.Datastore.UpdateDeviceTx(ctx, tx, host.Device) err = ds.UpdateDeviceTx(ctx, tx, host.Device)
if err != nil { if err != nil {
return ctxerr.Wrap(ctx, err, "update Android device") return ctxerr.Wrap(ctx, err, "update Android device")
} }

View file

@ -18,7 +18,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func TestHosts(t *testing.T) { func TestAndroidDevices(t *testing.T) {
ds := CreateMySQLDS(t) ds := CreateMySQLDS(t)
cases := []struct { cases := []struct {
@ -26,6 +26,7 @@ func TestHosts(t *testing.T) {
fn func(t *testing.T, ds *Datastore) fn func(t *testing.T, ds *Datastore)
}{ }{
{"CreateGetDevice", testCreateGetDevice}, {"CreateGetDevice", testCreateGetDevice},
{"UpdateDevice", testUpdateDevice},
} }
for _, c := range cases { for _, c := range cases {
t.Run(c.name, func(t *testing.T) { t.Run(c.name, func(t *testing.T) {
@ -74,8 +75,48 @@ func testCreateGetDevice(t *testing.T, ds *Datastore) {
assert.EqualValues(t, device2, result2) assert.EqualValues(t, device2, result2)
} }
func testUpdateDevice(t *testing.T, ds *Datastore) {
// Create initial device
device := &android.Device{
HostID: 1,
DeviceID: "deviceID",
EnterpriseSpecificID: ptr.String("enterpriseSpecificID"),
AndroidPolicyID: nil,
LastPolicySyncTime: nil,
}
created, err := ds.createDevice(testCtx(), device)
require.NoError(t, err)
// Update device
created.AndroidPolicyID = ptr.Uint(5)
created.LastPolicySyncTime = ptr.Time(time.Now().UTC().Truncate(time.Millisecond))
err = ds.updateDevice(testCtx(), created)
require.NoError(t, err)
// Verify update
result, err := ds.getDeviceByDeviceID(testCtx(), created.DeviceID)
require.NoError(t, err)
assert.Equal(t, created, result)
}
// Helper functions that use the Android datastore methods through the main datastore
func (ds *Datastore) createDevice(ctx context.Context, device *android.Device) (*android.Device, error) { func (ds *Datastore) createDevice(ctx context.Context, device *android.Device) (*android.Device, error) {
return ds.CreateDeviceTx(ctx, ds.Writer(ctx), device) err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error {
var err error
device, err = ds.CreateDeviceTx(ctx, tx, device)
return err
})
if err != nil {
return nil, err
}
return device, nil
}
func (ds *Datastore) updateDevice(ctx context.Context, device *android.Device) error {
return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error {
return ds.UpdateDeviceTx(ctx, tx, device)
})
} }
func (ds *Datastore) getDeviceByDeviceID(ctx context.Context, deviceID string) (*android.Device, error) { func (ds *Datastore) getDeviceByDeviceID(ctx context.Context, deviceID string) (*android.Device, error) {

View file

@ -1,7 +1,6 @@
package mysql package mysql
import ( import (
"context"
"testing" "testing"
"github.com/fleetdm/fleet/v4/server/datastore/mysql/common_mysql/testing_utils" "github.com/fleetdm/fleet/v4/server/datastore/mysql/common_mysql/testing_utils"
@ -11,7 +10,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func TestEnterprise(t *testing.T) { func TestAndroidEnterprises(t *testing.T) {
ds := CreateMySQLDS(t) ds := CreateMySQLDS(t)
cases := []struct { cases := []struct {
@ -20,7 +19,8 @@ func TestEnterprise(t *testing.T) {
}{ }{
{"CreateGetEnterprise", testCreateGetEnterprise}, {"CreateGetEnterprise", testCreateGetEnterprise},
{"UpdateEnterprise", testUpdateEnterprise}, {"UpdateEnterprise", testUpdateEnterprise},
{"DeleteAllEnterprises", testDeleteEnterprises}, {"DeleteEnterprises", testDeleteEnterprises},
{"GetEnterpriseBySignupToken", testGetEnterpriseBySignupToken},
} }
for _, c := range cases { for _, c := range cases {
t.Run(c.name, func(t *testing.T) { t.Run(c.name, func(t *testing.T) {
@ -93,7 +93,7 @@ func testDeleteEnterprises(t *testing.T, ds *Datastore) {
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, enterprise, result) assert.Equal(t, enterprise, result)
// Create enteprise without enterprise_id // Create enterprise without enterprise_id
id, err := ds.CreateEnterprise(testCtx(), 10) id, err := ds.CreateEnterprise(testCtx(), 10)
require.NoError(t, err) require.NoError(t, err)
assert.NotZero(t, id) assert.NotZero(t, id)
@ -120,7 +120,17 @@ func testDeleteEnterprises(t *testing.T, ds *Datastore) {
require.NoError(t, err) require.NoError(t, err)
_, err = ds.GetEnterpriseByID(testCtx(), enterprise.ID) _, err = ds.GetEnterpriseByID(testCtx(), enterprise.ID)
assert.True(t, fleet.IsNotFound(err)) assert.True(t, fleet.IsNotFound(err))
}
func testGetEnterpriseBySignupToken(t *testing.T, ds *Datastore) {
_, err := ds.GetEnterpriseBySignupToken(testCtx(), "nonexistent")
assert.True(t, fleet.IsNotFound(err))
enterprise := createEnterprise(t, ds)
result, err := ds.GetEnterpriseBySignupToken(testCtx(), enterprise.SignupToken)
require.NoError(t, err)
assert.Equal(t, enterprise, result)
} }
func createEnterprise(t *testing.T, ds *Datastore) *android.EnterpriseDetails { func createEnterprise(t *testing.T, ds *Datastore) *android.EnterpriseDetails {
@ -129,7 +139,9 @@ func createEnterprise(t *testing.T, ds *Datastore) *android.EnterpriseDetails {
ID: 9999, // start with an invalid ID ID: 9999, // start with an invalid ID
EnterpriseID: "LC04bp524j", EnterpriseID: "LC04bp524j",
}, },
SignupName: "signupUrls/C97372c91c6a85139", SignupName: "signupUrls/C97372c91c6a85139",
TopicID: "topicId",
SignupToken: "signupToken",
} }
const userID = uint(10) const userID = uint(10)
id, err := ds.CreateEnterprise(testCtx(), userID) id, err := ds.CreateEnterprise(testCtx(), userID)
@ -142,7 +154,3 @@ func createEnterprise(t *testing.T, ds *Datastore) *android.EnterpriseDetails {
require.NoError(t, err) require.NoError(t, err)
return enterprise return enterprise
} }
func testCtx() context.Context {
return context.Background()
}

View file

@ -11,7 +11,7 @@ import (
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
) )
func (ds *Datastore) CreateEnterprise(ctx context.Context, userID uint) (uint, error) { func (ds *AndroidDatastore) CreateEnterprise(ctx context.Context, userID uint) (uint, error) {
// android_enterprises user_id is only set when the row is created // android_enterprises user_id is only set when the row is created
stmt := `INSERT INTO android_enterprises (signup_name, user_id) VALUES ('', ?)` stmt := `INSERT INTO android_enterprises (signup_name, user_id) VALUES ('', ?)`
res, err := ds.Writer(ctx).ExecContext(ctx, stmt, userID) res, err := ds.Writer(ctx).ExecContext(ctx, stmt, userID)
@ -22,7 +22,7 @@ func (ds *Datastore) CreateEnterprise(ctx context.Context, userID uint) (uint, e
return uint(id), nil // nolint:gosec // dismiss G115 return uint(id), nil // nolint:gosec // dismiss G115
} }
func (ds *Datastore) GetEnterpriseByID(ctx context.Context, id uint) (*android.EnterpriseDetails, error) { func (ds *AndroidDatastore) GetEnterpriseByID(ctx context.Context, id uint) (*android.EnterpriseDetails, error) {
stmt := `SELECT id, signup_name, enterprise_id, pubsub_topic_id, signup_token, user_id FROM android_enterprises WHERE id = ?` stmt := `SELECT id, signup_name, enterprise_id, pubsub_topic_id, signup_token, user_id FROM android_enterprises WHERE id = ?`
var enterprise android.EnterpriseDetails var enterprise android.EnterpriseDetails
err := sqlx.GetContext(ctx, ds.reader(ctx), &enterprise, stmt, id) err := sqlx.GetContext(ctx, ds.reader(ctx), &enterprise, stmt, id)
@ -35,7 +35,7 @@ func (ds *Datastore) GetEnterpriseByID(ctx context.Context, id uint) (*android.E
return &enterprise, nil return &enterprise, nil
} }
func (ds *Datastore) GetEnterpriseBySignupToken(ctx context.Context, signupToken string) (*android.EnterpriseDetails, error) { func (ds *AndroidDatastore) GetEnterpriseBySignupToken(ctx context.Context, signupToken string) (*android.EnterpriseDetails, error) {
stmt := `SELECT id, signup_name, enterprise_id, pubsub_topic_id, signup_token, user_id FROM android_enterprises WHERE signup_token = ?` stmt := `SELECT id, signup_name, enterprise_id, pubsub_topic_id, signup_token, user_id FROM android_enterprises WHERE signup_token = ?`
var enterprise android.EnterpriseDetails var enterprise android.EnterpriseDetails
err := sqlx.GetContext(ctx, ds.reader(ctx), &enterprise, stmt, signupToken) err := sqlx.GetContext(ctx, ds.reader(ctx), &enterprise, stmt, signupToken)
@ -48,7 +48,7 @@ func (ds *Datastore) GetEnterpriseBySignupToken(ctx context.Context, signupToken
return &enterprise, nil return &enterprise, nil
} }
func (ds *Datastore) GetEnterprise(ctx context.Context) (*android.Enterprise, error) { func (ds *AndroidDatastore) GetEnterprise(ctx context.Context) (*android.Enterprise, error) {
stmt := `SELECT id, enterprise_id FROM android_enterprises WHERE enterprise_id != '' LIMIT 1` stmt := `SELECT id, enterprise_id FROM android_enterprises WHERE enterprise_id != '' LIMIT 1`
var enterprise android.Enterprise var enterprise android.Enterprise
err := sqlx.GetContext(ctx, ds.reader(ctx), &enterprise, stmt) err := sqlx.GetContext(ctx, ds.reader(ctx), &enterprise, stmt)
@ -61,7 +61,7 @@ func (ds *Datastore) GetEnterprise(ctx context.Context) (*android.Enterprise, er
return &enterprise, nil return &enterprise, nil
} }
func (ds *Datastore) UpdateEnterprise(ctx context.Context, enterprise *android.EnterpriseDetails) error { func (ds *AndroidDatastore) UpdateEnterprise(ctx context.Context, enterprise *android.EnterpriseDetails) error {
if enterprise == nil || enterprise.ID == 0 { if enterprise == nil || enterprise.ID == 0 {
return errors.New("missing enterprise ID") return errors.New("missing enterprise ID")
} }
@ -82,7 +82,7 @@ func (ds *Datastore) UpdateEnterprise(ctx context.Context, enterprise *android.E
return nil return nil
} }
func (ds *Datastore) DeleteOtherEnterprises(ctx context.Context, id uint) error { func (ds *AndroidDatastore) DeleteOtherEnterprises(ctx context.Context, id uint) error {
stmt := `DELETE FROM android_enterprises WHERE id != ?` stmt := `DELETE FROM android_enterprises WHERE id != ?`
_, err := ds.Writer(ctx).ExecContext(ctx, stmt, id) _, err := ds.Writer(ctx).ExecContext(ctx, stmt, id)
if err != nil { if err != nil {
@ -91,7 +91,7 @@ func (ds *Datastore) DeleteOtherEnterprises(ctx context.Context, id uint) error
return nil return nil
} }
func (ds *Datastore) DeleteAllEnterprises(ctx context.Context) error { func (ds *AndroidDatastore) DeleteAllEnterprises(ctx context.Context) error {
stmt := `DELETE FROM android_enterprises` stmt := `DELETE FROM android_enterprises`
_, err := ds.Writer(ctx).ExecContext(ctx, stmt) _, err := ds.Writer(ctx).ExecContext(ctx, stmt)
if err != nil { if err != nil {

View file

@ -9,7 +9,7 @@ import (
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
) )
func (ds *Datastore) CreateDeviceTx(ctx context.Context, tx sqlx.ExtContext, device *android.Device) (*android.Device, error) { func (ds *AndroidDatastore) CreateDeviceTx(ctx context.Context, tx sqlx.ExtContext, device *android.Device) (*android.Device, error) {
// Check for existing devices and duplicates // Check for existing devices and duplicates
stmt := `SELECT id, device_id, enterprise_specific_id FROM android_devices WHERE device_id = ? OR enterprise_specific_id = ?` stmt := `SELECT id, device_id, enterprise_specific_id FROM android_devices WHERE device_id = ? OR enterprise_specific_id = ?`
var existing []android.Device var existing []android.Device
@ -36,7 +36,7 @@ func (ds *Datastore) CreateDeviceTx(ctx context.Context, tx sqlx.ExtContext, dev
} }
} }
func (ds *Datastore) deleteDuplicate(ctx context.Context, device *android.Device, tx sqlx.ExtContext, existing []android.Device) error { func (ds *AndroidDatastore) deleteDuplicate(ctx context.Context, device *android.Device, tx sqlx.ExtContext, existing []android.Device) error {
// Duplicates should never happen. We log error and try to handle it gracefully. // Duplicates should never happen. We log error and try to handle it gracefully.
level.Error(ds.logger).Log("msg", "Found two Android devices with the same device ID or enterprise specific ID", "device_id", level.Error(ds.logger).Log("msg", "Found two Android devices with the same device ID or enterprise specific ID", "device_id",
device.DeviceID, "enterprise_specific_id", device.EnterpriseSpecificID) device.DeviceID, "enterprise_specific_id", device.EnterpriseSpecificID)
@ -49,13 +49,13 @@ func (ds *Datastore) deleteDuplicate(ctx context.Context, device *android.Device
return nil return nil
} }
func (ds *Datastore) deleteDevice(ctx context.Context, tx sqlx.ExtContext, id uint) error { func (ds *AndroidDatastore) deleteDevice(ctx context.Context, tx sqlx.ExtContext, id uint) error {
deleteStmt := `DELETE FROM android_devices WHERE id = ?` deleteStmt := `DELETE FROM android_devices WHERE id = ?`
_, err := tx.ExecContext(ctx, deleteStmt, id) _, err := tx.ExecContext(ctx, deleteStmt, id)
return err return err
} }
func (ds *Datastore) insertDevice(ctx context.Context, device *android.Device, tx sqlx.ExtContext) (*android.Device, error) { func (ds *AndroidDatastore) insertDevice(ctx context.Context, device *android.Device, tx sqlx.ExtContext) (*android.Device, error) {
stmt := `INSERT INTO android_devices (host_id, device_id, enterprise_specific_id, android_policy_id, last_policy_sync_time) VALUES (?, ?, ?, ?, stmt := `INSERT INTO android_devices (host_id, device_id, enterprise_specific_id, android_policy_id, last_policy_sync_time) VALUES (?, ?, ?, ?,
?)` ?)`
result, err := tx.ExecContext(ctx, stmt, device.HostID, device.DeviceID, device.EnterpriseSpecificID, device.AndroidPolicyID, result, err := tx.ExecContext(ctx, stmt, device.HostID, device.DeviceID, device.EnterpriseSpecificID, device.AndroidPolicyID,
@ -71,7 +71,7 @@ func (ds *Datastore) insertDevice(ctx context.Context, device *android.Device, t
return device, nil return device, nil
} }
func (ds *Datastore) updateDevice(ctx context.Context, device *android.Device, tx sqlx.ExtContext) (*android.Device, error) { func (ds *AndroidDatastore) updateDevice(ctx context.Context, device *android.Device, tx sqlx.ExtContext) (*android.Device, error) {
stmt := ` stmt := `
UPDATE android_devices SET UPDATE android_devices SET
host_id = :host_id, host_id = :host_id,
@ -91,7 +91,7 @@ func (ds *Datastore) updateDevice(ctx context.Context, device *android.Device, t
return device, nil return device, nil
} }
func (ds *Datastore) UpdateDeviceTx(ctx context.Context, tx sqlx.ExtContext, device *android.Device) error { func (ds *AndroidDatastore) UpdateDeviceTx(ctx context.Context, tx sqlx.ExtContext, device *android.Device) error {
_, err := ds.updateDevice(ctx, device, tx) _, err := ds.updateDevice(ctx, device, tx)
return err return err
} }

View file

@ -12,16 +12,16 @@ import (
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
) )
// Datastore is an implementation of android.Datastore interface backed by MySQL // AndroidDatastore is an implementation of android.Datastore interface backed by MySQL
type Datastore struct { type AndroidDatastore struct {
logger log.Logger logger log.Logger
primary *sqlx.DB primary *sqlx.DB
replica fleet.DBReader // so it cannot be used to perform writes replica fleet.DBReader // so it cannot be used to perform writes
} }
// New creates a new Datastore // NewAndroidDatastore creates a new Android Datastore
func New(logger log.Logger, primary *sqlx.DB, replica fleet.DBReader) android.Datastore { func NewAndroidDatastore(logger log.Logger, primary *sqlx.DB, replica fleet.DBReader) android.Datastore {
return &Datastore{ return &AndroidDatastore{
logger: logger, logger: logger,
primary: primary, primary: primary,
replica: replica, replica: replica,
@ -31,7 +31,7 @@ func New(logger log.Logger, primary *sqlx.DB, replica fleet.DBReader) android.Da
// reader returns the DB instance to use for read-only statements, which is the // reader returns the DB instance to use for read-only statements, which is the
// replica unless the primary has been explicitly required via // replica unless the primary has been explicitly required via
// ctxdb.RequirePrimary. // ctxdb.RequirePrimary.
func (ds *Datastore) reader(ctx context.Context) fleet.DBReader { func (ds *AndroidDatastore) reader(ctx context.Context) fleet.DBReader {
if ctxdb.IsPrimaryRequired(ctx) { if ctxdb.IsPrimaryRequired(ctx) {
return ds.primary return ds.primary
} }
@ -40,10 +40,10 @@ func (ds *Datastore) reader(ctx context.Context) fleet.DBReader {
// Writer returns the DB instance to use for write statements, which is always // Writer returns the DB instance to use for write statements, which is always
// the primary. // the primary.
func (ds *Datastore) Writer(_ context.Context) *sqlx.DB { func (ds *AndroidDatastore) Writer(_ context.Context) *sqlx.DB {
return ds.primary return ds.primary
} }
func (ds *Datastore) WithRetryTxx(ctx context.Context, fn common_mysql.TxFn) (err error) { func (ds *AndroidDatastore) WithRetryTxx(ctx context.Context, fn common_mysql.TxFn) (err error) {
return common_mysql.WithRetryTxx(ctx, ds.Writer(ctx), fn, ds.logger) return common_mysql.WithRetryTxx(ctx, ds.Writer(ctx), fn, ds.logger)
} }

View file

@ -27,7 +27,6 @@ import (
"github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/goose" "github.com/fleetdm/fleet/v4/server/goose"
"github.com/fleetdm/fleet/v4/server/mdm/android" "github.com/fleetdm/fleet/v4/server/mdm/android"
android_mysql "github.com/fleetdm/fleet/v4/server/mdm/android/mysql"
nano_push "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/push" nano_push "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/push"
scep_depot "github.com/fleetdm/fleet/v4/server/mdm/scep/depot" scep_depot "github.com/fleetdm/fleet/v4/server/mdm/scep/depot"
"github.com/go-kit/log" "github.com/go-kit/log"
@ -266,7 +265,7 @@ func New(config config.MysqlConfig, c clock.Clock, opts ...DBOption) (*Datastore
stmtCache: make(map[string]*sqlx.Stmt), stmtCache: make(map[string]*sqlx.Stmt),
minLastOpenedAtDiff: options.MinLastOpenedAtDiff, minLastOpenedAtDiff: options.MinLastOpenedAtDiff,
serverPrivateKey: options.PrivateKey, serverPrivateKey: options.PrivateKey,
Datastore: android_mysql.New(options.Logger, dbWriter, dbReader), Datastore: NewAndroidDatastore(options.Logger, dbWriter, dbReader),
} }
go ds.writeChanLoop() go ds.writeChanLoop()

View file

@ -24,10 +24,11 @@ func TestAllAndroidPackageDependencies(t *testing.T) {
"github.com/fleetdm/fleet/v4/server/service/middleware/endpoint_utils", "github.com/fleetdm/fleet/v4/server/service/middleware/endpoint_utils",
"github.com/fleetdm/fleet/v4/server/service/middleware/log", "github.com/fleetdm/fleet/v4/server/service/middleware/log",
"github.com/fleetdm/fleet/v4/server/service/middleware/ratelimit", "github.com/fleetdm/fleet/v4/server/service/middleware/ratelimit",
"github.com/fleetdm/fleet/v4/server/mdm/android/tests...", // Android functionality moved to main datastore
). ).
ShouldNotDependOn( ShouldNotDependOn(
"github.com/fleetdm/fleet/v4/server/service...", "github.com/fleetdm/fleet/v4/server/service...",
"github.com/fleetdm/fleet/v4/server/datastore...", "github.com/fleetdm/fleet/v4/server/datastore/mysql...",
) )
} }

View file

@ -1,4 +1,3 @@
package mock package mock
//go:generate go run ../../../mock/mockimpl/impl.go -o client.go "p *Client" "androidmgmt.Client" //go:generate go run ../../../mock/mockimpl/impl.go -o client.go "p *Client" "androidmgmt.Client"
//go:generate go run ../../../mock/mockimpl/impl.go -o datastore.go "ds *Datastore" "android.Datastore"

View file

@ -1,125 +0,0 @@
// Automatically generated by mockimpl. DO NOT EDIT!
package mock
import (
"context"
"sync"
"github.com/fleetdm/fleet/v4/server/mdm/android"
"github.com/jmoiron/sqlx"
)
var _ android.Datastore = (*Datastore)(nil)
type CreateEnterpriseFunc func(ctx context.Context, userID uint) (uint, error)
type GetEnterpriseByIDFunc func(ctx context.Context, ID uint) (*android.EnterpriseDetails, error)
type GetEnterpriseBySignupTokenFunc func(ctx context.Context, signupToken string) (*android.EnterpriseDetails, error)
type GetEnterpriseFunc func(ctx context.Context) (*android.Enterprise, error)
type UpdateEnterpriseFunc func(ctx context.Context, enterprise *android.EnterpriseDetails) error
type DeleteAllEnterprisesFunc func(ctx context.Context) error
type DeleteOtherEnterprisesFunc func(ctx context.Context, ID uint) error
type CreateDeviceTxFunc func(ctx context.Context, tx sqlx.ExtContext, device *android.Device) (*android.Device, error)
type UpdateDeviceTxFunc func(ctx context.Context, tx sqlx.ExtContext, device *android.Device) error
type Datastore struct {
CreateEnterpriseFunc CreateEnterpriseFunc
CreateEnterpriseFuncInvoked bool
GetEnterpriseByIDFunc GetEnterpriseByIDFunc
GetEnterpriseByIDFuncInvoked bool
GetEnterpriseBySignupTokenFunc GetEnterpriseBySignupTokenFunc
GetEnterpriseBySignupTokenFuncInvoked bool
GetEnterpriseFunc GetEnterpriseFunc
GetEnterpriseFuncInvoked bool
UpdateEnterpriseFunc UpdateEnterpriseFunc
UpdateEnterpriseFuncInvoked bool
DeleteAllEnterprisesFunc DeleteAllEnterprisesFunc
DeleteAllEnterprisesFuncInvoked bool
DeleteOtherEnterprisesFunc DeleteOtherEnterprisesFunc
DeleteOtherEnterprisesFuncInvoked bool
CreateDeviceTxFunc CreateDeviceTxFunc
CreateDeviceTxFuncInvoked bool
UpdateDeviceTxFunc UpdateDeviceTxFunc
UpdateDeviceTxFuncInvoked bool
mu sync.Mutex
}
func (ds *Datastore) CreateEnterprise(ctx context.Context, userID uint) (uint, error) {
ds.mu.Lock()
ds.CreateEnterpriseFuncInvoked = true
ds.mu.Unlock()
return ds.CreateEnterpriseFunc(ctx, userID)
}
func (ds *Datastore) GetEnterpriseByID(ctx context.Context, ID uint) (*android.EnterpriseDetails, error) {
ds.mu.Lock()
ds.GetEnterpriseByIDFuncInvoked = true
ds.mu.Unlock()
return ds.GetEnterpriseByIDFunc(ctx, ID)
}
func (ds *Datastore) GetEnterpriseBySignupToken(ctx context.Context, signupToken string) (*android.EnterpriseDetails, error) {
ds.mu.Lock()
ds.GetEnterpriseBySignupTokenFuncInvoked = true
ds.mu.Unlock()
return ds.GetEnterpriseBySignupTokenFunc(ctx, signupToken)
}
func (ds *Datastore) GetEnterprise(ctx context.Context) (*android.Enterprise, error) {
ds.mu.Lock()
ds.GetEnterpriseFuncInvoked = true
ds.mu.Unlock()
return ds.GetEnterpriseFunc(ctx)
}
func (ds *Datastore) UpdateEnterprise(ctx context.Context, enterprise *android.EnterpriseDetails) error {
ds.mu.Lock()
ds.UpdateEnterpriseFuncInvoked = true
ds.mu.Unlock()
return ds.UpdateEnterpriseFunc(ctx, enterprise)
}
func (ds *Datastore) DeleteAllEnterprises(ctx context.Context) error {
ds.mu.Lock()
ds.DeleteAllEnterprisesFuncInvoked = true
ds.mu.Unlock()
return ds.DeleteAllEnterprisesFunc(ctx)
}
func (ds *Datastore) DeleteOtherEnterprises(ctx context.Context, ID uint) error {
ds.mu.Lock()
ds.DeleteOtherEnterprisesFuncInvoked = true
ds.mu.Unlock()
return ds.DeleteOtherEnterprisesFunc(ctx, ID)
}
func (ds *Datastore) CreateDeviceTx(ctx context.Context, tx sqlx.ExtContext, device *android.Device) (*android.Device, error) {
ds.mu.Lock()
ds.CreateDeviceTxFuncInvoked = true
ds.mu.Unlock()
return ds.CreateDeviceTxFunc(ctx, tx, device)
}
func (ds *Datastore) UpdateDeviceTx(ctx context.Context, tx sqlx.ExtContext, device *android.Device) error {
ds.mu.Lock()
ds.UpdateDeviceTxFuncInvoked = true
ds.mu.Unlock()
return ds.UpdateDeviceTxFunc(ctx, tx, device)
}

View file

@ -1,43 +0,0 @@
package mock
import (
"context"
"errors"
"github.com/fleetdm/fleet/v4/server/mdm/android"
)
func (s *Datastore) InitCommonMocks() {
s.CreateEnterpriseFunc = func(ctx context.Context, _ uint) (uint, error) {
return 1, nil
}
s.UpdateEnterpriseFunc = func(ctx context.Context, enterprise *android.EnterpriseDetails) error {
return nil
}
s.GetEnterpriseFunc = func(ctx context.Context) (*android.Enterprise, error) {
return &android.Enterprise{}, nil
}
s.GetEnterpriseByIDFunc = func(ctx context.Context, ID uint) (*android.EnterpriseDetails, error) {
return &android.EnterpriseDetails{}, nil
}
s.GetEnterpriseBySignupTokenFunc = func(ctx context.Context, signupToken string) (*android.EnterpriseDetails, error) {
if signupToken == "signup_token" {
return &android.EnterpriseDetails{}, nil
}
return nil, &notFoundError{errors.New("not found")}
}
s.DeleteAllEnterprisesFunc = func(ctx context.Context) error {
return nil
}
s.DeleteOtherEnterprisesFunc = func(ctx context.Context, ID uint) error {
return nil
}
}
type notFoundError struct {
error
}
func (e *notFoundError) IsNotFound() bool {
return true
}

View file

@ -1,31 +0,0 @@
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `android_enterprises` (
`id` int unsigned NOT NULL AUTO_INCREMENT,
`signup_name` varchar(63) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '',
`enterprise_id` varchar(63) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '',
`created_at` datetime(6) DEFAULT CURRENT_TIMESTAMP(6),
`updated_at` datetime(6) DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
`signup_token` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '',
`pubsub_topic_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '',
`user_id` int unsigned NOT NULL DEFAULT '0',
PRIMARY KEY (`id`)
) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `android_devices` (
`id` int unsigned NOT NULL AUTO_INCREMENT,
`host_id` int unsigned NOT NULL,
`device_id` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
`enterprise_specific_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`android_policy_id` int unsigned DEFAULT NULL,
`last_policy_sync_time` datetime(3) DEFAULT NULL,
`created_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
`updated_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
PRIMARY KEY (`id`),
UNIQUE KEY `idx_android_devices_host_id` (`host_id`),
UNIQUE KEY `idx_android_devices_device_id` (`device_id`),
UNIQUE KEY `idx_android_devices_enterprise_specific_id` (`enterprise_specific_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

View file

@ -1,54 +0,0 @@
package mysql
import (
"os"
"path"
"runtime"
"testing"
"github.com/fleetdm/fleet/v4/server/datastore/mysql/common_mysql"
"github.com/fleetdm/fleet/v4/server/datastore/mysql/common_mysql/testing_utils"
"github.com/go-kit/log"
"github.com/stretchr/testify/require"
)
// Android MySQL testing utilities. This file should contain VERY LITTLE code since it is also compiled into the production binary.
// Whenever possible, new code should go into a dedicated testing package (e.g. mdm/android/mysql/tests/testing_utils.go).
// These utilities are used to create a MySQL Datastore for testing the Android MDM MySQL implementation.
// They are located in the same package as the implementation to prevent a circular dependency. If put it in a different package,
// the circular dependency would be: mysql -> testing_utils -> mysql
func CreateMySQLDS(t testing.TB) *Datastore {
return createMySQLDSWithOptions(t, nil)
}
func createMySQLDSWithOptions(t testing.TB, opts *testing_utils.DatastoreTestOptions) *Datastore {
cleanTestName, opts := testing_utils.ProcessOptions(t, opts)
ds := InitializeDatabase(t, cleanTestName, opts)
t.Cleanup(func() { Close(ds) })
return ds
}
// InitializeDatabase loads the dumped schema into a newly created database in MySQL.
// This is much faster than running the full set of migrations on each test.
func InitializeDatabase(t testing.TB, testName string, opts *testing_utils.DatastoreTestOptions) *Datastore {
_, filename, _, _ := runtime.Caller(0)
schemaPath := path.Join(path.Dir(filename), "schema.sql")
testing_utils.LoadSchema(t, testName, opts, schemaPath)
return connectMySQL(t, testName)
}
func connectMySQL(t testing.TB, testName string) *Datastore {
// Import TestSQLMode from main MySQL testing utils to ensure consistent SQL modes across all tests
// This ensures Android tests catch the same data integrity issues as other MySQL tests
dbWriter, err := common_mysql.NewDB(testing_utils.MysqlTestConfig(testName), &common_mysql.DBOptions{
SqlMode: common_mysql.TestSQLMode,
}, "")
require.NoError(t, err)
ds := New(log.NewLogfmtLogger(os.Stdout), dbWriter, dbWriter)
return ds.(*Datastore)
}
func Close(ds *Datastore) {
_ = ds.primary.Close()
}

View file

@ -8,6 +8,7 @@ import (
"github.com/fleetdm/fleet/v4/server/authz" "github.com/fleetdm/fleet/v4/server/authz"
"github.com/fleetdm/fleet/v4/server/contexts/viewer" "github.com/fleetdm/fleet/v4/server/contexts/viewer"
"github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/mdm/android"
android_mock "github.com/fleetdm/fleet/v4/server/mdm/android/mock" android_mock "github.com/fleetdm/fleet/v4/server/mdm/android/mock"
ds_mock "github.com/fleetdm/fleet/v4/server/mock" ds_mock "github.com/fleetdm/fleet/v4/server/mock"
"github.com/fleetdm/fleet/v4/server/ptr" "github.com/fleetdm/fleet/v4/server/ptr"
@ -151,7 +152,31 @@ func checkAuthErr(t *testing.T, shouldFail bool, err error) {
func InitCommonDSMocks() *AndroidMockDS { func InitCommonDSMocks() *AndroidMockDS {
ds := AndroidMockDS{} ds := AndroidMockDS{}
ds.Datastore.InitCommonMocks() // Set up basic Android datastore mocks directly on the Fleet datastore mock
ds.Store.CreateEnterpriseFunc = func(ctx context.Context, _ uint) (uint, error) {
return 1, nil
}
ds.Store.UpdateEnterpriseFunc = func(ctx context.Context, enterprise *android.EnterpriseDetails) error {
return nil
}
ds.Store.GetEnterpriseFunc = func(ctx context.Context) (*android.Enterprise, error) {
return &android.Enterprise{}, nil
}
ds.Store.GetEnterpriseByIDFunc = func(ctx context.Context, ID uint) (*android.EnterpriseDetails, error) {
return &android.EnterpriseDetails{}, nil
}
ds.Store.GetEnterpriseBySignupTokenFunc = func(ctx context.Context, signupToken string) (*android.EnterpriseDetails, error) {
if signupToken == "signup_token" {
return &android.EnterpriseDetails{}, nil
}
return nil, &notFoundError{}
}
ds.Store.DeleteAllEnterprisesFunc = func(ctx context.Context) error {
return nil
}
ds.Store.DeleteOtherEnterprisesFunc = func(ctx context.Context, ID uint) error {
return nil
}
ds.Store.AppConfigFunc = func(_ context.Context) (*fleet.AppConfig, error) { ds.Store.AppConfigFunc = func(_ context.Context) (*fleet.AppConfig, error) {
return &fleet.AppConfig{}, nil return &fleet.AppConfig{}, nil
@ -184,10 +209,14 @@ func InitCommonDSMocks() *AndroidMockDS {
} }
type AndroidMockDS struct { type AndroidMockDS struct {
android_mock.Datastore
ds_mock.Store ds_mock.Store
} }
type notFoundError struct{}
func (e *notFoundError) Error() string { return "not found" }
func (e *notFoundError) IsNotFound() bool { return true }
type mockService struct { type mockService struct {
mock.Mock mock.Mock
fleet.Service fleet.Service

View file

@ -9,11 +9,10 @@ import (
"testing" "testing"
"github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/config"
"github.com/fleetdm/fleet/v4/server/datastore/mysql/common_mysql/testing_utils" "github.com/fleetdm/fleet/v4/server/datastore/mysql"
"github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/mdm/android" "github.com/fleetdm/fleet/v4/server/mdm/android"
android_mock "github.com/fleetdm/fleet/v4/server/mdm/android/mock" android_mock "github.com/fleetdm/fleet/v4/server/mdm/android/mock"
"github.com/fleetdm/fleet/v4/server/mdm/android/mysql"
"github.com/fleetdm/fleet/v4/server/mdm/android/service" "github.com/fleetdm/fleet/v4/server/mdm/android/service"
"github.com/fleetdm/fleet/v4/server/mdm/android/service/androidmgmt" "github.com/fleetdm/fleet/v4/server/mdm/android/service/androidmgmt"
ds_mock "github.com/fleetdm/fleet/v4/server/mock" ds_mock "github.com/fleetdm/fleet/v4/server/mock"
@ -42,6 +41,46 @@ type AndroidDSWithMock struct {
ds_mock.Store ds_mock.Store
} }
// resolve ambiguity between embedded datastore and mock methods
func (ds *AndroidDSWithMock) AppConfig(ctx context.Context) (*fleet.AppConfig, error) {
return ds.Store.AppConfig(ctx) // use mock datastore
}
func (ds *AndroidDSWithMock) CreateDeviceTx(ctx context.Context, tx sqlx.ExtContext, device *android.Device) (*android.Device, error) {
return ds.Datastore.CreateDeviceTx(ctx, tx, device)
}
func (ds *AndroidDSWithMock) UpdateDeviceTx(ctx context.Context, tx sqlx.ExtContext, device *android.Device) error {
return ds.Datastore.UpdateDeviceTx(ctx, tx, device)
}
func (ds *AndroidDSWithMock) CreateEnterprise(ctx context.Context, userID uint) (uint, error) {
return ds.Datastore.CreateEnterprise(ctx, userID)
}
func (ds *AndroidDSWithMock) GetEnterpriseByID(ctx context.Context, id uint) (*android.EnterpriseDetails, error) {
return ds.Datastore.GetEnterpriseByID(ctx, id)
}
func (ds *AndroidDSWithMock) GetEnterpriseBySignupToken(ctx context.Context, signupToken string) (*android.EnterpriseDetails, error) {
return ds.Datastore.GetEnterpriseBySignupToken(ctx, signupToken)
}
func (ds *AndroidDSWithMock) GetEnterprise(ctx context.Context) (*android.Enterprise, error) {
return ds.Datastore.GetEnterprise(ctx)
}
func (ds *AndroidDSWithMock) UpdateEnterprise(ctx context.Context, enterprise *android.EnterpriseDetails) error {
return ds.Datastore.UpdateEnterprise(ctx, enterprise)
}
func (ds *AndroidDSWithMock) DeleteAllEnterprises(ctx context.Context) error {
return ds.Datastore.DeleteAllEnterprises(ctx)
}
func (ds *AndroidDSWithMock) DeleteOtherEnterprises(ctx context.Context, id uint) error {
return ds.Datastore.DeleteOtherEnterprises(ctx, id)
}
type WithServer struct { type WithServer struct {
suite.Suite suite.Suite
Svc android.Service Svc android.Service
@ -135,7 +174,7 @@ func (ts *WithServer) createCommonProxyMocks(t *testing.T) {
} }
func (ts *WithServer) TearDownSuite() { func (ts *WithServer) TearDownSuite() {
mysql.Close(ts.DS.Datastore) ts.DS.Datastore.Close()
} }
type mockService struct { type mockService struct {
@ -194,7 +233,6 @@ func CreateNamedMySQLDS(t *testing.T, name string) *mysql.Datastore {
if _, ok := os.LookupEnv("MYSQL_TEST"); !ok { if _, ok := os.LookupEnv("MYSQL_TEST"); !ok {
t.Skip("MySQL tests are disabled") t.Skip("MySQL tests are disabled")
} }
ds := mysql.InitializeDatabase(t, name, new(testing_utils.DatastoreTestOptions)) // use the standard Fleet datastore for Android integration tests
t.Cleanup(func() { mysql.Close(ds) }) return mysql.CreateMySQLDS(t)
return ds
} }

View file

@ -24,7 +24,6 @@ func createHttpClient(m dsl.Matcher) {
func txCheck(m dsl.Matcher) { func txCheck(m dsl.Matcher) {
m.Import("github.com/fleetdm/fleet/v4/server/datastore/mysql") m.Import("github.com/fleetdm/fleet/v4/server/datastore/mysql")
m.Import("github.com/fleetdm/fleet/v4/server/mdm/android/mysql")
m.Import("github.com/jmoiron/sqlx") m.Import("github.com/jmoiron/sqlx")
isDatastoreType := func(v dsl.Var) bool { isDatastoreType := func(v dsl.Var) bool {

View file

@ -12,7 +12,6 @@ import (
"github.com/WatchBeam/clock" "github.com/WatchBeam/clock"
"github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/config"
"github.com/fleetdm/fleet/v4/server/datastore/mysql" "github.com/fleetdm/fleet/v4/server/datastore/mysql"
"github.com/fleetdm/fleet/v4/server/mdm/android"
"github.com/go-kit/log" "github.com/go-kit/log"
) )
@ -38,15 +37,13 @@ func panicif(err error) {
} }
} }
// main requires 2 arguments: // main requires 1 argument:
// 1. Path to dumpfile // 1. Path to dumpfile
// 2. Path to Android dumpfile
func main() { func main() {
if len(os.Args) != 3 { if len(os.Args) != 2 {
panic("not enough arguments") panic("not enough arguments")
} }
fmt.Println("dumping schema to", os.Args[1]) fmt.Println("dumping schema to", os.Args[1])
fmt.Println("dumping Android schema to", os.Args[2])
// Create the database (must use raw MySQL client to do this) // Create the database (must use raw MySQL client to do this)
db, err := sql.Open( db, err := sql.Open(
@ -91,16 +88,4 @@ func main() {
panicif(cmd.Run()) panicif(cmd.Run())
panicif(os.WriteFile(os.Args[1], stdoutBuf.Bytes(), 0o655)) panicif(os.WriteFile(os.Args[1], stdoutBuf.Bytes(), 0o655))
// Dump Android schema
args := []string{"compose", "exec", "-T", "mysql_test"}
// Command to run inside container:
args = append(args, "mysqldump", "-u"+testUsername, "-p"+testPassword, "schemadb")
args = append(args, android.MySQLTables()...)
args = append(args, "--compact", "--skip-comments")
cmd = exec.Command("docker", args...)
stdoutBuf = bytes.Buffer{}
cmd.Stdout = &stdoutBuf
panicif(cmd.Run())
panicif(os.WriteFile(os.Args[2], stdoutBuf.Bytes(), 0o655))
} }