From 8bc8d01f0a52342678ac7d0159ac5f0ccbb28337 Mon Sep 17 00:00:00 2001 From: Carlo <1778532+cdcme@users.noreply.github.com> Date: Mon, 25 Aug 2025 11:41:28 -0400 Subject: [PATCH] Merge Android datastore into main Fleet datastore (#32233) Resolves #31218 --- .github/workflows/test-db-changes.yml | 3 +- Makefile | 4 +- git-hooks/backend/hooks/db-schema | 2 +- server/datastore/mysql/android.go | 4 +- .../mysql/android_device_test.go} | 45 ++++++- .../mysql/android_enterprise_test.go} | 26 ++-- .../mysql/android_enterprises.go} | 14 +- .../mysql/android_hosts.go} | 12 +- .../mysql/android_mysql.go} | 16 +-- server/datastore/mysql/mysql.go | 3 +- server/mdm/android/arch_test.go | 3 +- server/mdm/android/mock/android.go | 1 - server/mdm/android/mock/datastore.go | 125 ------------------ server/mdm/android/mock/datastore_setup.go | 43 ------ server/mdm/android/mysql/schema.sql | 31 ----- server/mdm/android/mysql/testing_utils.go | 54 -------- .../mdm/android/service/enterprises_test.go | 33 ++++- server/mdm/android/tests/testing_utils.go | 50 ++++++- tools/ci/rules.go | 1 - tools/dbutils/schema_generator.go | 19 +-- 20 files changed, 167 insertions(+), 322 deletions(-) rename server/{mdm/android/mysql/hosts_test.go => datastore/mysql/android_device_test.go} (68%) rename server/{mdm/android/mysql/enterprises_test.go => datastore/mysql/android_enterprise_test.go} (85%) rename server/{mdm/android/mysql/enterprises.go => datastore/mysql/android_enterprises.go} (80%) rename server/{mdm/android/mysql/hosts.go => datastore/mysql/android_hosts.go} (79%) rename server/{mdm/android/mysql/mysql.go => datastore/mysql/android_mysql.go} (64%) delete mode 100644 server/mdm/android/mock/datastore.go delete mode 100644 server/mdm/android/mock/datastore_setup.go delete mode 100644 server/mdm/android/mysql/schema.sql delete mode 100644 server/mdm/android/mysql/testing_utils.go diff --git a/.github/workflows/test-db-changes.yml b/.github/workflows/test-db-changes.yml index 13f467e3d3..b73d181b6c 100644 --- a/.github/workflows/test-db-changes.yml +++ b/.github/workflows/test-db-changes.yml @@ -10,7 +10,6 @@ on: paths: - '**.go' - 'server/datastore/mysql/schema.sql' - - 'server/mdm/android/mysql/schema.sql' - '.github/workflows/test-db-changes.yml' workflow_dispatch: # Manual @@ -93,7 +92,7 @@ jobs: - name: Verify test schema changes run: | make test-schema - if [[ $(git diff-files --patch server/datastore/mysql/schema.sql server/mdm/android/mysql/schema.sql) ]]; then + if [[ $(git diff-files --patch server/datastore/mysql/schema.sql) ]]; then echo "❌ fail: uncommited changes in schema.sql" echo "please run 'make test-schema' and commit the changes" exit 1 diff --git a/Makefile b/Makefile index 49038ed5b2..bb141e2922 100644 --- a/Makefile +++ b/Makefile @@ -232,7 +232,7 @@ endif .help-short--test-schema: @echo "Update schema.sql from current migrations" test-schema: - go run ./tools/dbutils ./server/datastore/mysql/schema.sql ./server/mdm/android/mysql/schema.sql + go run ./tools/dbutils ./server/datastore/mysql/schema.sql dump-test-schema: test-schema # This is the base command to run Go tests. @@ -307,7 +307,7 @@ FAST_PKGS_TO_TEST := \ ./server/mdm/scep/x509util \ ./server/policies FLEETCTL_PKGS_TO_TEST := ./cmd/fleetctl/... -MYSQL_PKGS_TO_TEST := ./server/datastore/mysql/... ./server/mdm/android/mysql +MYSQL_PKGS_TO_TEST := ./server/datastore/mysql/... SCRIPTS_PKGS_TO_TEST := ./orbit/pkg/scripts SERVICE_PKGS_TO_TEST := ./server/service VULN_PKGS_TO_TEST := ./server/vulnerabilities/... diff --git a/git-hooks/backend/hooks/db-schema b/git-hooks/backend/hooks/db-schema index dda2df14ac..7f4439effe 100644 --- a/git-hooks/backend/hooks/db-schema +++ b/git-hooks/backend/hooks/db-schema @@ -3,7 +3,7 @@ ## DB Schema Dump if [[ $DB_SCHEMA_DUMP ]]; then make dump-test-schema - if [[ $(git diff-files --patch server/datastore/mysql/schema.sql server/mdm/android/mysql/schema.sql) ]]; then + if [[ $(git diff-files --patch server/datastore/mysql/schema.sql) ]]; then echo "❌ fail: uncommited changes in schema.sql" echo "please run 'make dump-test-schema' and commit the changes" exit 1 diff --git a/server/datastore/mysql/android.go b/server/datastore/mysql/android.go index 5ad4a12d00..5d41642fb6 100644 --- a/server/datastore/mysql/android.go +++ b/server/datastore/mysql/android.go @@ -116,7 +116,7 @@ func (ds *Datastore) NewAndroidHost(ctx context.Context, host *fleet.AndroidHost return ctxerr.Wrap(ctx, err, "new Android host MDM info") } - host.Device, err = ds.Datastore.CreateDeviceTx(ctx, tx, host.Device) + host.Device, err = ds.CreateDeviceTx(ctx, tx, host.Device) if err != nil { return ctxerr.Wrap(ctx, err, "creating new Android device") } @@ -203,7 +203,7 @@ func (ds *Datastore) UpdateAndroidHost(ctx context.Context, host *fleet.AndroidH } } - err = ds.Datastore.UpdateDeviceTx(ctx, tx, host.Device) + err = ds.UpdateDeviceTx(ctx, tx, host.Device) if err != nil { return ctxerr.Wrap(ctx, err, "update Android device") } diff --git a/server/mdm/android/mysql/hosts_test.go b/server/datastore/mysql/android_device_test.go similarity index 68% rename from server/mdm/android/mysql/hosts_test.go rename to server/datastore/mysql/android_device_test.go index 50ed842b70..fae4456ed8 100644 --- a/server/mdm/android/mysql/hosts_test.go +++ b/server/datastore/mysql/android_device_test.go @@ -18,7 +18,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestHosts(t *testing.T) { +func TestAndroidDevices(t *testing.T) { ds := CreateMySQLDS(t) cases := []struct { @@ -26,6 +26,7 @@ func TestHosts(t *testing.T) { fn func(t *testing.T, ds *Datastore) }{ {"CreateGetDevice", testCreateGetDevice}, + {"UpdateDevice", testUpdateDevice}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -74,8 +75,48 @@ func testCreateGetDevice(t *testing.T, ds *Datastore) { assert.EqualValues(t, device2, result2) } +func testUpdateDevice(t *testing.T, ds *Datastore) { + // Create initial device + device := &android.Device{ + HostID: 1, + DeviceID: "deviceID", + EnterpriseSpecificID: ptr.String("enterpriseSpecificID"), + AndroidPolicyID: nil, + LastPolicySyncTime: nil, + } + created, err := ds.createDevice(testCtx(), device) + require.NoError(t, err) + + // Update device + created.AndroidPolicyID = ptr.Uint(5) + created.LastPolicySyncTime = ptr.Time(time.Now().UTC().Truncate(time.Millisecond)) + + err = ds.updateDevice(testCtx(), created) + require.NoError(t, err) + + // Verify update + result, err := ds.getDeviceByDeviceID(testCtx(), created.DeviceID) + require.NoError(t, err) + assert.Equal(t, created, result) +} + +// Helper functions that use the Android datastore methods through the main datastore func (ds *Datastore) createDevice(ctx context.Context, device *android.Device) (*android.Device, error) { - return ds.CreateDeviceTx(ctx, ds.Writer(ctx), device) + err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { + var err error + device, err = ds.CreateDeviceTx(ctx, tx, device) + return err + }) + if err != nil { + return nil, err + } + return device, nil +} + +func (ds *Datastore) updateDevice(ctx context.Context, device *android.Device) error { + return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { + return ds.UpdateDeviceTx(ctx, tx, device) + }) } func (ds *Datastore) getDeviceByDeviceID(ctx context.Context, deviceID string) (*android.Device, error) { diff --git a/server/mdm/android/mysql/enterprises_test.go b/server/datastore/mysql/android_enterprise_test.go similarity index 85% rename from server/mdm/android/mysql/enterprises_test.go rename to server/datastore/mysql/android_enterprise_test.go index 76f5b6ebed..e88d9620b2 100644 --- a/server/mdm/android/mysql/enterprises_test.go +++ b/server/datastore/mysql/android_enterprise_test.go @@ -1,7 +1,6 @@ package mysql import ( - "context" "testing" "github.com/fleetdm/fleet/v4/server/datastore/mysql/common_mysql/testing_utils" @@ -11,7 +10,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestEnterprise(t *testing.T) { +func TestAndroidEnterprises(t *testing.T) { ds := CreateMySQLDS(t) cases := []struct { @@ -20,7 +19,8 @@ func TestEnterprise(t *testing.T) { }{ {"CreateGetEnterprise", testCreateGetEnterprise}, {"UpdateEnterprise", testUpdateEnterprise}, - {"DeleteAllEnterprises", testDeleteEnterprises}, + {"DeleteEnterprises", testDeleteEnterprises}, + {"GetEnterpriseBySignupToken", testGetEnterpriseBySignupToken}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -93,7 +93,7 @@ func testDeleteEnterprises(t *testing.T, ds *Datastore) { require.NoError(t, err) assert.Equal(t, enterprise, result) - // Create enteprise without enterprise_id + // Create enterprise without enterprise_id id, err := ds.CreateEnterprise(testCtx(), 10) require.NoError(t, err) assert.NotZero(t, id) @@ -120,7 +120,17 @@ func testDeleteEnterprises(t *testing.T, ds *Datastore) { require.NoError(t, err) _, err = ds.GetEnterpriseByID(testCtx(), enterprise.ID) assert.True(t, fleet.IsNotFound(err)) +} +func testGetEnterpriseBySignupToken(t *testing.T, ds *Datastore) { + _, err := ds.GetEnterpriseBySignupToken(testCtx(), "nonexistent") + assert.True(t, fleet.IsNotFound(err)) + + enterprise := createEnterprise(t, ds) + + result, err := ds.GetEnterpriseBySignupToken(testCtx(), enterprise.SignupToken) + require.NoError(t, err) + assert.Equal(t, enterprise, result) } func createEnterprise(t *testing.T, ds *Datastore) *android.EnterpriseDetails { @@ -129,7 +139,9 @@ func createEnterprise(t *testing.T, ds *Datastore) *android.EnterpriseDetails { ID: 9999, // start with an invalid ID EnterpriseID: "LC04bp524j", }, - SignupName: "signupUrls/C97372c91c6a85139", + SignupName: "signupUrls/C97372c91c6a85139", + TopicID: "topicId", + SignupToken: "signupToken", } const userID = uint(10) id, err := ds.CreateEnterprise(testCtx(), userID) @@ -142,7 +154,3 @@ func createEnterprise(t *testing.T, ds *Datastore) *android.EnterpriseDetails { require.NoError(t, err) return enterprise } - -func testCtx() context.Context { - return context.Background() -} diff --git a/server/mdm/android/mysql/enterprises.go b/server/datastore/mysql/android_enterprises.go similarity index 80% rename from server/mdm/android/mysql/enterprises.go rename to server/datastore/mysql/android_enterprises.go index 1b5a1951a7..eeff53b80f 100644 --- a/server/mdm/android/mysql/enterprises.go +++ b/server/datastore/mysql/android_enterprises.go @@ -11,7 +11,7 @@ import ( "github.com/jmoiron/sqlx" ) -func (ds *Datastore) CreateEnterprise(ctx context.Context, userID uint) (uint, error) { +func (ds *AndroidDatastore) CreateEnterprise(ctx context.Context, userID uint) (uint, error) { // android_enterprises user_id is only set when the row is created stmt := `INSERT INTO android_enterprises (signup_name, user_id) VALUES ('', ?)` res, err := ds.Writer(ctx).ExecContext(ctx, stmt, userID) @@ -22,7 +22,7 @@ func (ds *Datastore) CreateEnterprise(ctx context.Context, userID uint) (uint, e return uint(id), nil // nolint:gosec // dismiss G115 } -func (ds *Datastore) GetEnterpriseByID(ctx context.Context, id uint) (*android.EnterpriseDetails, error) { +func (ds *AndroidDatastore) GetEnterpriseByID(ctx context.Context, id uint) (*android.EnterpriseDetails, error) { stmt := `SELECT id, signup_name, enterprise_id, pubsub_topic_id, signup_token, user_id FROM android_enterprises WHERE id = ?` var enterprise android.EnterpriseDetails err := sqlx.GetContext(ctx, ds.reader(ctx), &enterprise, stmt, id) @@ -35,7 +35,7 @@ func (ds *Datastore) GetEnterpriseByID(ctx context.Context, id uint) (*android.E return &enterprise, nil } -func (ds *Datastore) GetEnterpriseBySignupToken(ctx context.Context, signupToken string) (*android.EnterpriseDetails, error) { +func (ds *AndroidDatastore) GetEnterpriseBySignupToken(ctx context.Context, signupToken string) (*android.EnterpriseDetails, error) { stmt := `SELECT id, signup_name, enterprise_id, pubsub_topic_id, signup_token, user_id FROM android_enterprises WHERE signup_token = ?` var enterprise android.EnterpriseDetails err := sqlx.GetContext(ctx, ds.reader(ctx), &enterprise, stmt, signupToken) @@ -48,7 +48,7 @@ func (ds *Datastore) GetEnterpriseBySignupToken(ctx context.Context, signupToken return &enterprise, nil } -func (ds *Datastore) GetEnterprise(ctx context.Context) (*android.Enterprise, error) { +func (ds *AndroidDatastore) GetEnterprise(ctx context.Context) (*android.Enterprise, error) { stmt := `SELECT id, enterprise_id FROM android_enterprises WHERE enterprise_id != '' LIMIT 1` var enterprise android.Enterprise err := sqlx.GetContext(ctx, ds.reader(ctx), &enterprise, stmt) @@ -61,7 +61,7 @@ func (ds *Datastore) GetEnterprise(ctx context.Context) (*android.Enterprise, er return &enterprise, nil } -func (ds *Datastore) UpdateEnterprise(ctx context.Context, enterprise *android.EnterpriseDetails) error { +func (ds *AndroidDatastore) UpdateEnterprise(ctx context.Context, enterprise *android.EnterpriseDetails) error { if enterprise == nil || enterprise.ID == 0 { return errors.New("missing enterprise ID") } @@ -82,7 +82,7 @@ func (ds *Datastore) UpdateEnterprise(ctx context.Context, enterprise *android.E return nil } -func (ds *Datastore) DeleteOtherEnterprises(ctx context.Context, id uint) error { +func (ds *AndroidDatastore) DeleteOtherEnterprises(ctx context.Context, id uint) error { stmt := `DELETE FROM android_enterprises WHERE id != ?` _, err := ds.Writer(ctx).ExecContext(ctx, stmt, id) if err != nil { @@ -91,7 +91,7 @@ func (ds *Datastore) DeleteOtherEnterprises(ctx context.Context, id uint) error return nil } -func (ds *Datastore) DeleteAllEnterprises(ctx context.Context) error { +func (ds *AndroidDatastore) DeleteAllEnterprises(ctx context.Context) error { stmt := `DELETE FROM android_enterprises` _, err := ds.Writer(ctx).ExecContext(ctx, stmt) if err != nil { diff --git a/server/mdm/android/mysql/hosts.go b/server/datastore/mysql/android_hosts.go similarity index 79% rename from server/mdm/android/mysql/hosts.go rename to server/datastore/mysql/android_hosts.go index d2354284b2..c4142bbdb6 100644 --- a/server/mdm/android/mysql/hosts.go +++ b/server/datastore/mysql/android_hosts.go @@ -9,7 +9,7 @@ import ( "github.com/jmoiron/sqlx" ) -func (ds *Datastore) CreateDeviceTx(ctx context.Context, tx sqlx.ExtContext, device *android.Device) (*android.Device, error) { +func (ds *AndroidDatastore) CreateDeviceTx(ctx context.Context, tx sqlx.ExtContext, device *android.Device) (*android.Device, error) { // Check for existing devices and duplicates stmt := `SELECT id, device_id, enterprise_specific_id FROM android_devices WHERE device_id = ? OR enterprise_specific_id = ?` var existing []android.Device @@ -36,7 +36,7 @@ func (ds *Datastore) CreateDeviceTx(ctx context.Context, tx sqlx.ExtContext, dev } } -func (ds *Datastore) deleteDuplicate(ctx context.Context, device *android.Device, tx sqlx.ExtContext, existing []android.Device) error { +func (ds *AndroidDatastore) deleteDuplicate(ctx context.Context, device *android.Device, tx sqlx.ExtContext, existing []android.Device) error { // Duplicates should never happen. We log error and try to handle it gracefully. level.Error(ds.logger).Log("msg", "Found two Android devices with the same device ID or enterprise specific ID", "device_id", device.DeviceID, "enterprise_specific_id", device.EnterpriseSpecificID) @@ -49,13 +49,13 @@ func (ds *Datastore) deleteDuplicate(ctx context.Context, device *android.Device return nil } -func (ds *Datastore) deleteDevice(ctx context.Context, tx sqlx.ExtContext, id uint) error { +func (ds *AndroidDatastore) deleteDevice(ctx context.Context, tx sqlx.ExtContext, id uint) error { deleteStmt := `DELETE FROM android_devices WHERE id = ?` _, err := tx.ExecContext(ctx, deleteStmt, id) return err } -func (ds *Datastore) insertDevice(ctx context.Context, device *android.Device, tx sqlx.ExtContext) (*android.Device, error) { +func (ds *AndroidDatastore) insertDevice(ctx context.Context, device *android.Device, tx sqlx.ExtContext) (*android.Device, error) { stmt := `INSERT INTO android_devices (host_id, device_id, enterprise_specific_id, android_policy_id, last_policy_sync_time) VALUES (?, ?, ?, ?, ?)` result, err := tx.ExecContext(ctx, stmt, device.HostID, device.DeviceID, device.EnterpriseSpecificID, device.AndroidPolicyID, @@ -71,7 +71,7 @@ func (ds *Datastore) insertDevice(ctx context.Context, device *android.Device, t return device, nil } -func (ds *Datastore) updateDevice(ctx context.Context, device *android.Device, tx sqlx.ExtContext) (*android.Device, error) { +func (ds *AndroidDatastore) updateDevice(ctx context.Context, device *android.Device, tx sqlx.ExtContext) (*android.Device, error) { stmt := ` UPDATE android_devices SET host_id = :host_id, @@ -91,7 +91,7 @@ func (ds *Datastore) updateDevice(ctx context.Context, device *android.Device, t return device, nil } -func (ds *Datastore) UpdateDeviceTx(ctx context.Context, tx sqlx.ExtContext, device *android.Device) error { +func (ds *AndroidDatastore) UpdateDeviceTx(ctx context.Context, tx sqlx.ExtContext, device *android.Device) error { _, err := ds.updateDevice(ctx, device, tx) return err } diff --git a/server/mdm/android/mysql/mysql.go b/server/datastore/mysql/android_mysql.go similarity index 64% rename from server/mdm/android/mysql/mysql.go rename to server/datastore/mysql/android_mysql.go index 8abe51e206..1c197e07be 100644 --- a/server/mdm/android/mysql/mysql.go +++ b/server/datastore/mysql/android_mysql.go @@ -12,16 +12,16 @@ import ( "github.com/jmoiron/sqlx" ) -// Datastore is an implementation of android.Datastore interface backed by MySQL -type Datastore struct { +// AndroidDatastore is an implementation of android.Datastore interface backed by MySQL +type AndroidDatastore struct { logger log.Logger primary *sqlx.DB replica fleet.DBReader // so it cannot be used to perform writes } -// New creates a new Datastore -func New(logger log.Logger, primary *sqlx.DB, replica fleet.DBReader) android.Datastore { - return &Datastore{ +// NewAndroidDatastore creates a new Android Datastore +func NewAndroidDatastore(logger log.Logger, primary *sqlx.DB, replica fleet.DBReader) android.Datastore { + return &AndroidDatastore{ logger: logger, primary: primary, replica: replica, @@ -31,7 +31,7 @@ func New(logger log.Logger, primary *sqlx.DB, replica fleet.DBReader) android.Da // reader returns the DB instance to use for read-only statements, which is the // replica unless the primary has been explicitly required via // ctxdb.RequirePrimary. -func (ds *Datastore) reader(ctx context.Context) fleet.DBReader { +func (ds *AndroidDatastore) reader(ctx context.Context) fleet.DBReader { if ctxdb.IsPrimaryRequired(ctx) { return ds.primary } @@ -40,10 +40,10 @@ func (ds *Datastore) reader(ctx context.Context) fleet.DBReader { // Writer returns the DB instance to use for write statements, which is always // the primary. -func (ds *Datastore) Writer(_ context.Context) *sqlx.DB { +func (ds *AndroidDatastore) Writer(_ context.Context) *sqlx.DB { return ds.primary } -func (ds *Datastore) WithRetryTxx(ctx context.Context, fn common_mysql.TxFn) (err error) { +func (ds *AndroidDatastore) WithRetryTxx(ctx context.Context, fn common_mysql.TxFn) (err error) { return common_mysql.WithRetryTxx(ctx, ds.Writer(ctx), fn, ds.logger) } diff --git a/server/datastore/mysql/mysql.go b/server/datastore/mysql/mysql.go index 1eb669ec30..876f3b0d10 100644 --- a/server/datastore/mysql/mysql.go +++ b/server/datastore/mysql/mysql.go @@ -27,7 +27,6 @@ import ( "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/goose" "github.com/fleetdm/fleet/v4/server/mdm/android" - android_mysql "github.com/fleetdm/fleet/v4/server/mdm/android/mysql" nano_push "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/push" scep_depot "github.com/fleetdm/fleet/v4/server/mdm/scep/depot" "github.com/go-kit/log" @@ -266,7 +265,7 @@ func New(config config.MysqlConfig, c clock.Clock, opts ...DBOption) (*Datastore stmtCache: make(map[string]*sqlx.Stmt), minLastOpenedAtDiff: options.MinLastOpenedAtDiff, serverPrivateKey: options.PrivateKey, - Datastore: android_mysql.New(options.Logger, dbWriter, dbReader), + Datastore: NewAndroidDatastore(options.Logger, dbWriter, dbReader), } go ds.writeChanLoop() diff --git a/server/mdm/android/arch_test.go b/server/mdm/android/arch_test.go index fd753ad881..7c4e91d592 100644 --- a/server/mdm/android/arch_test.go +++ b/server/mdm/android/arch_test.go @@ -24,10 +24,11 @@ func TestAllAndroidPackageDependencies(t *testing.T) { "github.com/fleetdm/fleet/v4/server/service/middleware/endpoint_utils", "github.com/fleetdm/fleet/v4/server/service/middleware/log", "github.com/fleetdm/fleet/v4/server/service/middleware/ratelimit", + "github.com/fleetdm/fleet/v4/server/mdm/android/tests...", // Android functionality moved to main datastore ). ShouldNotDependOn( "github.com/fleetdm/fleet/v4/server/service...", - "github.com/fleetdm/fleet/v4/server/datastore...", + "github.com/fleetdm/fleet/v4/server/datastore/mysql...", ) } diff --git a/server/mdm/android/mock/android.go b/server/mdm/android/mock/android.go index 14908f0055..17ac4ef9d8 100644 --- a/server/mdm/android/mock/android.go +++ b/server/mdm/android/mock/android.go @@ -1,4 +1,3 @@ package mock //go:generate go run ../../../mock/mockimpl/impl.go -o client.go "p *Client" "androidmgmt.Client" -//go:generate go run ../../../mock/mockimpl/impl.go -o datastore.go "ds *Datastore" "android.Datastore" diff --git a/server/mdm/android/mock/datastore.go b/server/mdm/android/mock/datastore.go deleted file mode 100644 index 1b34d36e24..0000000000 --- a/server/mdm/android/mock/datastore.go +++ /dev/null @@ -1,125 +0,0 @@ -// Automatically generated by mockimpl. DO NOT EDIT! - -package mock - -import ( - "context" - "sync" - - "github.com/fleetdm/fleet/v4/server/mdm/android" - "github.com/jmoiron/sqlx" -) - -var _ android.Datastore = (*Datastore)(nil) - -type CreateEnterpriseFunc func(ctx context.Context, userID uint) (uint, error) - -type GetEnterpriseByIDFunc func(ctx context.Context, ID uint) (*android.EnterpriseDetails, error) - -type GetEnterpriseBySignupTokenFunc func(ctx context.Context, signupToken string) (*android.EnterpriseDetails, error) - -type GetEnterpriseFunc func(ctx context.Context) (*android.Enterprise, error) - -type UpdateEnterpriseFunc func(ctx context.Context, enterprise *android.EnterpriseDetails) error - -type DeleteAllEnterprisesFunc func(ctx context.Context) error - -type DeleteOtherEnterprisesFunc func(ctx context.Context, ID uint) error - -type CreateDeviceTxFunc func(ctx context.Context, tx sqlx.ExtContext, device *android.Device) (*android.Device, error) - -type UpdateDeviceTxFunc func(ctx context.Context, tx sqlx.ExtContext, device *android.Device) error - -type Datastore struct { - CreateEnterpriseFunc CreateEnterpriseFunc - CreateEnterpriseFuncInvoked bool - - GetEnterpriseByIDFunc GetEnterpriseByIDFunc - GetEnterpriseByIDFuncInvoked bool - - GetEnterpriseBySignupTokenFunc GetEnterpriseBySignupTokenFunc - GetEnterpriseBySignupTokenFuncInvoked bool - - GetEnterpriseFunc GetEnterpriseFunc - GetEnterpriseFuncInvoked bool - - UpdateEnterpriseFunc UpdateEnterpriseFunc - UpdateEnterpriseFuncInvoked bool - - DeleteAllEnterprisesFunc DeleteAllEnterprisesFunc - DeleteAllEnterprisesFuncInvoked bool - - DeleteOtherEnterprisesFunc DeleteOtherEnterprisesFunc - DeleteOtherEnterprisesFuncInvoked bool - - CreateDeviceTxFunc CreateDeviceTxFunc - CreateDeviceTxFuncInvoked bool - - UpdateDeviceTxFunc UpdateDeviceTxFunc - UpdateDeviceTxFuncInvoked bool - - mu sync.Mutex -} - -func (ds *Datastore) CreateEnterprise(ctx context.Context, userID uint) (uint, error) { - ds.mu.Lock() - ds.CreateEnterpriseFuncInvoked = true - ds.mu.Unlock() - return ds.CreateEnterpriseFunc(ctx, userID) -} - -func (ds *Datastore) GetEnterpriseByID(ctx context.Context, ID uint) (*android.EnterpriseDetails, error) { - ds.mu.Lock() - ds.GetEnterpriseByIDFuncInvoked = true - ds.mu.Unlock() - return ds.GetEnterpriseByIDFunc(ctx, ID) -} - -func (ds *Datastore) GetEnterpriseBySignupToken(ctx context.Context, signupToken string) (*android.EnterpriseDetails, error) { - ds.mu.Lock() - ds.GetEnterpriseBySignupTokenFuncInvoked = true - ds.mu.Unlock() - return ds.GetEnterpriseBySignupTokenFunc(ctx, signupToken) -} - -func (ds *Datastore) GetEnterprise(ctx context.Context) (*android.Enterprise, error) { - ds.mu.Lock() - ds.GetEnterpriseFuncInvoked = true - ds.mu.Unlock() - return ds.GetEnterpriseFunc(ctx) -} - -func (ds *Datastore) UpdateEnterprise(ctx context.Context, enterprise *android.EnterpriseDetails) error { - ds.mu.Lock() - ds.UpdateEnterpriseFuncInvoked = true - ds.mu.Unlock() - return ds.UpdateEnterpriseFunc(ctx, enterprise) -} - -func (ds *Datastore) DeleteAllEnterprises(ctx context.Context) error { - ds.mu.Lock() - ds.DeleteAllEnterprisesFuncInvoked = true - ds.mu.Unlock() - return ds.DeleteAllEnterprisesFunc(ctx) -} - -func (ds *Datastore) DeleteOtherEnterprises(ctx context.Context, ID uint) error { - ds.mu.Lock() - ds.DeleteOtherEnterprisesFuncInvoked = true - ds.mu.Unlock() - return ds.DeleteOtherEnterprisesFunc(ctx, ID) -} - -func (ds *Datastore) CreateDeviceTx(ctx context.Context, tx sqlx.ExtContext, device *android.Device) (*android.Device, error) { - ds.mu.Lock() - ds.CreateDeviceTxFuncInvoked = true - ds.mu.Unlock() - return ds.CreateDeviceTxFunc(ctx, tx, device) -} - -func (ds *Datastore) UpdateDeviceTx(ctx context.Context, tx sqlx.ExtContext, device *android.Device) error { - ds.mu.Lock() - ds.UpdateDeviceTxFuncInvoked = true - ds.mu.Unlock() - return ds.UpdateDeviceTxFunc(ctx, tx, device) -} diff --git a/server/mdm/android/mock/datastore_setup.go b/server/mdm/android/mock/datastore_setup.go deleted file mode 100644 index 702bdf73d0..0000000000 --- a/server/mdm/android/mock/datastore_setup.go +++ /dev/null @@ -1,43 +0,0 @@ -package mock - -import ( - "context" - "errors" - - "github.com/fleetdm/fleet/v4/server/mdm/android" -) - -func (s *Datastore) InitCommonMocks() { - s.CreateEnterpriseFunc = func(ctx context.Context, _ uint) (uint, error) { - return 1, nil - } - s.UpdateEnterpriseFunc = func(ctx context.Context, enterprise *android.EnterpriseDetails) error { - return nil - } - s.GetEnterpriseFunc = func(ctx context.Context) (*android.Enterprise, error) { - return &android.Enterprise{}, nil - } - s.GetEnterpriseByIDFunc = func(ctx context.Context, ID uint) (*android.EnterpriseDetails, error) { - return &android.EnterpriseDetails{}, nil - } - s.GetEnterpriseBySignupTokenFunc = func(ctx context.Context, signupToken string) (*android.EnterpriseDetails, error) { - if signupToken == "signup_token" { - return &android.EnterpriseDetails{}, nil - } - return nil, ¬FoundError{errors.New("not found")} - } - s.DeleteAllEnterprisesFunc = func(ctx context.Context) error { - return nil - } - s.DeleteOtherEnterprisesFunc = func(ctx context.Context, ID uint) error { - return nil - } -} - -type notFoundError struct { - error -} - -func (e *notFoundError) IsNotFound() bool { - return true -} diff --git a/server/mdm/android/mysql/schema.sql b/server/mdm/android/mysql/schema.sql deleted file mode 100644 index 34d5ce89b9..0000000000 --- a/server/mdm/android/mysql/schema.sql +++ /dev/null @@ -1,31 +0,0 @@ -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!50503 SET character_set_client = utf8mb4 */; -CREATE TABLE `android_enterprises` ( - `id` int unsigned NOT NULL AUTO_INCREMENT, - `signup_name` varchar(63) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', - `enterprise_id` varchar(63) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', - `created_at` datetime(6) DEFAULT CURRENT_TIMESTAMP(6), - `updated_at` datetime(6) DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), - `signup_token` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', - `pubsub_topic_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', - `user_id` int unsigned NOT NULL DEFAULT '0', - PRIMARY KEY (`id`) -) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -/*!40101 SET character_set_client = @saved_cs_client */; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!50503 SET character_set_client = utf8mb4 */; -CREATE TABLE `android_devices` ( - `id` int unsigned NOT NULL AUTO_INCREMENT, - `host_id` int unsigned NOT NULL, - `device_id` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, - `enterprise_specific_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL, - `android_policy_id` int unsigned DEFAULT NULL, - `last_policy_sync_time` datetime(3) DEFAULT NULL, - `created_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), - `updated_at` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), - PRIMARY KEY (`id`), - UNIQUE KEY `idx_android_devices_host_id` (`host_id`), - UNIQUE KEY `idx_android_devices_device_id` (`device_id`), - UNIQUE KEY `idx_android_devices_enterprise_specific_id` (`enterprise_specific_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -/*!40101 SET character_set_client = @saved_cs_client */; diff --git a/server/mdm/android/mysql/testing_utils.go b/server/mdm/android/mysql/testing_utils.go deleted file mode 100644 index 1963402feb..0000000000 --- a/server/mdm/android/mysql/testing_utils.go +++ /dev/null @@ -1,54 +0,0 @@ -package mysql - -import ( - "os" - "path" - "runtime" - "testing" - - "github.com/fleetdm/fleet/v4/server/datastore/mysql/common_mysql" - "github.com/fleetdm/fleet/v4/server/datastore/mysql/common_mysql/testing_utils" - "github.com/go-kit/log" - "github.com/stretchr/testify/require" -) - -// Android MySQL testing utilities. This file should contain VERY LITTLE code since it is also compiled into the production binary. -// Whenever possible, new code should go into a dedicated testing package (e.g. mdm/android/mysql/tests/testing_utils.go). -// These utilities are used to create a MySQL Datastore for testing the Android MDM MySQL implementation. -// They are located in the same package as the implementation to prevent a circular dependency. If put it in a different package, -// the circular dependency would be: mysql -> testing_utils -> mysql - -func CreateMySQLDS(t testing.TB) *Datastore { - return createMySQLDSWithOptions(t, nil) -} - -func createMySQLDSWithOptions(t testing.TB, opts *testing_utils.DatastoreTestOptions) *Datastore { - cleanTestName, opts := testing_utils.ProcessOptions(t, opts) - ds := InitializeDatabase(t, cleanTestName, opts) - t.Cleanup(func() { Close(ds) }) - return ds -} - -// InitializeDatabase loads the dumped schema into a newly created database in MySQL. -// This is much faster than running the full set of migrations on each test. -func InitializeDatabase(t testing.TB, testName string, opts *testing_utils.DatastoreTestOptions) *Datastore { - _, filename, _, _ := runtime.Caller(0) - schemaPath := path.Join(path.Dir(filename), "schema.sql") - testing_utils.LoadSchema(t, testName, opts, schemaPath) - return connectMySQL(t, testName) -} - -func connectMySQL(t testing.TB, testName string) *Datastore { - // Import TestSQLMode from main MySQL testing utils to ensure consistent SQL modes across all tests - // This ensures Android tests catch the same data integrity issues as other MySQL tests - dbWriter, err := common_mysql.NewDB(testing_utils.MysqlTestConfig(testName), &common_mysql.DBOptions{ - SqlMode: common_mysql.TestSQLMode, - }, "") - require.NoError(t, err) - ds := New(log.NewLogfmtLogger(os.Stdout), dbWriter, dbWriter) - return ds.(*Datastore) -} - -func Close(ds *Datastore) { - _ = ds.primary.Close() -} diff --git a/server/mdm/android/service/enterprises_test.go b/server/mdm/android/service/enterprises_test.go index 5a1cdb91e7..19cb24da5a 100644 --- a/server/mdm/android/service/enterprises_test.go +++ b/server/mdm/android/service/enterprises_test.go @@ -8,6 +8,7 @@ import ( "github.com/fleetdm/fleet/v4/server/authz" "github.com/fleetdm/fleet/v4/server/contexts/viewer" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/mdm/android" android_mock "github.com/fleetdm/fleet/v4/server/mdm/android/mock" ds_mock "github.com/fleetdm/fleet/v4/server/mock" "github.com/fleetdm/fleet/v4/server/ptr" @@ -151,7 +152,31 @@ func checkAuthErr(t *testing.T, shouldFail bool, err error) { func InitCommonDSMocks() *AndroidMockDS { ds := AndroidMockDS{} - ds.Datastore.InitCommonMocks() + // Set up basic Android datastore mocks directly on the Fleet datastore mock + ds.Store.CreateEnterpriseFunc = func(ctx context.Context, _ uint) (uint, error) { + return 1, nil + } + ds.Store.UpdateEnterpriseFunc = func(ctx context.Context, enterprise *android.EnterpriseDetails) error { + return nil + } + ds.Store.GetEnterpriseFunc = func(ctx context.Context) (*android.Enterprise, error) { + return &android.Enterprise{}, nil + } + ds.Store.GetEnterpriseByIDFunc = func(ctx context.Context, ID uint) (*android.EnterpriseDetails, error) { + return &android.EnterpriseDetails{}, nil + } + ds.Store.GetEnterpriseBySignupTokenFunc = func(ctx context.Context, signupToken string) (*android.EnterpriseDetails, error) { + if signupToken == "signup_token" { + return &android.EnterpriseDetails{}, nil + } + return nil, ¬FoundError{} + } + ds.Store.DeleteAllEnterprisesFunc = func(ctx context.Context) error { + return nil + } + ds.Store.DeleteOtherEnterprisesFunc = func(ctx context.Context, ID uint) error { + return nil + } ds.Store.AppConfigFunc = func(_ context.Context) (*fleet.AppConfig, error) { return &fleet.AppConfig{}, nil @@ -184,10 +209,14 @@ func InitCommonDSMocks() *AndroidMockDS { } type AndroidMockDS struct { - android_mock.Datastore ds_mock.Store } +type notFoundError struct{} + +func (e *notFoundError) Error() string { return "not found" } +func (e *notFoundError) IsNotFound() bool { return true } + type mockService struct { mock.Mock fleet.Service diff --git a/server/mdm/android/tests/testing_utils.go b/server/mdm/android/tests/testing_utils.go index b76a4cfce4..90ae5b4beb 100644 --- a/server/mdm/android/tests/testing_utils.go +++ b/server/mdm/android/tests/testing_utils.go @@ -9,11 +9,10 @@ import ( "testing" "github.com/fleetdm/fleet/v4/server/config" - "github.com/fleetdm/fleet/v4/server/datastore/mysql/common_mysql/testing_utils" + "github.com/fleetdm/fleet/v4/server/datastore/mysql" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/mdm/android" android_mock "github.com/fleetdm/fleet/v4/server/mdm/android/mock" - "github.com/fleetdm/fleet/v4/server/mdm/android/mysql" "github.com/fleetdm/fleet/v4/server/mdm/android/service" "github.com/fleetdm/fleet/v4/server/mdm/android/service/androidmgmt" ds_mock "github.com/fleetdm/fleet/v4/server/mock" @@ -42,6 +41,46 @@ type AndroidDSWithMock struct { ds_mock.Store } +// resolve ambiguity between embedded datastore and mock methods +func (ds *AndroidDSWithMock) AppConfig(ctx context.Context) (*fleet.AppConfig, error) { + return ds.Store.AppConfig(ctx) // use mock datastore +} +func (ds *AndroidDSWithMock) CreateDeviceTx(ctx context.Context, tx sqlx.ExtContext, device *android.Device) (*android.Device, error) { + return ds.Datastore.CreateDeviceTx(ctx, tx, device) +} + +func (ds *AndroidDSWithMock) UpdateDeviceTx(ctx context.Context, tx sqlx.ExtContext, device *android.Device) error { + return ds.Datastore.UpdateDeviceTx(ctx, tx, device) +} + +func (ds *AndroidDSWithMock) CreateEnterprise(ctx context.Context, userID uint) (uint, error) { + return ds.Datastore.CreateEnterprise(ctx, userID) +} + +func (ds *AndroidDSWithMock) GetEnterpriseByID(ctx context.Context, id uint) (*android.EnterpriseDetails, error) { + return ds.Datastore.GetEnterpriseByID(ctx, id) +} + +func (ds *AndroidDSWithMock) GetEnterpriseBySignupToken(ctx context.Context, signupToken string) (*android.EnterpriseDetails, error) { + return ds.Datastore.GetEnterpriseBySignupToken(ctx, signupToken) +} + +func (ds *AndroidDSWithMock) GetEnterprise(ctx context.Context) (*android.Enterprise, error) { + return ds.Datastore.GetEnterprise(ctx) +} + +func (ds *AndroidDSWithMock) UpdateEnterprise(ctx context.Context, enterprise *android.EnterpriseDetails) error { + return ds.Datastore.UpdateEnterprise(ctx, enterprise) +} + +func (ds *AndroidDSWithMock) DeleteAllEnterprises(ctx context.Context) error { + return ds.Datastore.DeleteAllEnterprises(ctx) +} + +func (ds *AndroidDSWithMock) DeleteOtherEnterprises(ctx context.Context, id uint) error { + return ds.Datastore.DeleteOtherEnterprises(ctx, id) +} + type WithServer struct { suite.Suite Svc android.Service @@ -135,7 +174,7 @@ func (ts *WithServer) createCommonProxyMocks(t *testing.T) { } func (ts *WithServer) TearDownSuite() { - mysql.Close(ts.DS.Datastore) + ts.DS.Datastore.Close() } type mockService struct { @@ -194,7 +233,6 @@ func CreateNamedMySQLDS(t *testing.T, name string) *mysql.Datastore { if _, ok := os.LookupEnv("MYSQL_TEST"); !ok { t.Skip("MySQL tests are disabled") } - ds := mysql.InitializeDatabase(t, name, new(testing_utils.DatastoreTestOptions)) - t.Cleanup(func() { mysql.Close(ds) }) - return ds + // use the standard Fleet datastore for Android integration tests + return mysql.CreateMySQLDS(t) } diff --git a/tools/ci/rules.go b/tools/ci/rules.go index 68b9500ea4..0ce02529bd 100644 --- a/tools/ci/rules.go +++ b/tools/ci/rules.go @@ -24,7 +24,6 @@ func createHttpClient(m dsl.Matcher) { func txCheck(m dsl.Matcher) { m.Import("github.com/fleetdm/fleet/v4/server/datastore/mysql") - m.Import("github.com/fleetdm/fleet/v4/server/mdm/android/mysql") m.Import("github.com/jmoiron/sqlx") isDatastoreType := func(v dsl.Var) bool { diff --git a/tools/dbutils/schema_generator.go b/tools/dbutils/schema_generator.go index de4f333d76..b7386b91bc 100644 --- a/tools/dbutils/schema_generator.go +++ b/tools/dbutils/schema_generator.go @@ -12,7 +12,6 @@ import ( "github.com/WatchBeam/clock" "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/datastore/mysql" - "github.com/fleetdm/fleet/v4/server/mdm/android" "github.com/go-kit/log" ) @@ -38,15 +37,13 @@ func panicif(err error) { } } -// main requires 2 arguments: +// main requires 1 argument: // 1. Path to dumpfile -// 2. Path to Android dumpfile func main() { - if len(os.Args) != 3 { + if len(os.Args) != 2 { panic("not enough arguments") } fmt.Println("dumping schema to", os.Args[1]) - fmt.Println("dumping Android schema to", os.Args[2]) // Create the database (must use raw MySQL client to do this) db, err := sql.Open( @@ -91,16 +88,4 @@ func main() { panicif(cmd.Run()) panicif(os.WriteFile(os.Args[1], stdoutBuf.Bytes(), 0o655)) - - // Dump Android schema - args := []string{"compose", "exec", "-T", "mysql_test"} - // Command to run inside container: - args = append(args, "mysqldump", "-u"+testUsername, "-p"+testPassword, "schemadb") - args = append(args, android.MySQLTables()...) - args = append(args, "--compact", "--skip-comments") - cmd = exec.Command("docker", args...) - stdoutBuf = bytes.Buffer{} - cmd.Stdout = &stdoutBuf - panicif(cmd.Run()) - panicif(os.WriteFile(os.Args[2], stdoutBuf.Bytes(), 0o655)) }