From e68535d468fa86d5b862ace684dff141c9bfcfd0 Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Fri, 9 Dec 2022 13:21:30 -0300 Subject: [PATCH] report errors that can occur during file carving (#8972) related to https://github.com/fleetdm/fleet/issues/8117 --- changes/8117-file-carving-errors | 1 + cmd/fleet/main.go | 11 ++ cmd/fleetctl/get.go | 24 ++-- cmd/fleetctl/get_test.go | 107 ++++++++++++++++++ docs/Using-Fleet/REST-API.md | 3 +- server/datastore/mysql/carves.go | 27 +++-- .../20221205112142_AddErrorToCarveMetadata.go | 20 ++++ ...1205112142_AddErrorToCarveMetadata_test.go | 53 +++++++++ .../mysql/migrations/tables/migration_test.go | 4 +- server/datastore/mysql/schema.sql | 5 +- server/datastore/s3/carves.go | 18 ++- server/datastore/s3/carves_test.go | 1 + server/fleet/carves.go | 2 + server/service/carves.go | 28 ++++- server/service/carves_test.go | 20 ++++ server/service/integration_core_test.go | 10 ++ 16 files changed, 308 insertions(+), 26 deletions(-) create mode 100644 changes/8117-file-carving-errors create mode 100644 server/datastore/mysql/migrations/tables/20221205112142_AddErrorToCarveMetadata.go create mode 100644 server/datastore/mysql/migrations/tables/20221205112142_AddErrorToCarveMetadata_test.go create mode 100644 server/datastore/s3/carves_test.go diff --git a/changes/8117-file-carving-errors b/changes/8117-file-carving-errors new file mode 100644 index 0000000000..35514913b3 --- /dev/null +++ b/changes/8117-file-carving-errors @@ -0,0 +1 @@ +- Added functionality to report if a carve failed along with its error message. diff --git a/cmd/fleet/main.go b/cmd/fleet/main.go index 9c4c142311..ee66e037a1 100644 --- a/cmd/fleet/main.go +++ b/cmd/fleet/main.go @@ -69,6 +69,17 @@ func applyDevFlags(cfg *config.FleetConfig) { cfg.Prometheus.BasicAuth.Password = "insecure" } + cfg.S3 = config.S3Config{ + Bucket: "carves-dev", + Region: "minio", + Prefix: "dev-prefix", + EndpointURL: "localhost:9000", + AccessKeyID: "minio", + SecretAccessKey: "minio123!", + DisableSSL: true, + ForceS3PathStyle: true, + } + cfg.Packaging.S3 = config.S3Config{ Bucket: "installers-dev", Region: "minio", diff --git a/cmd/fleetctl/get.go b/cmd/fleetctl/get.go index c7c5038a15..e61a266f12 100644 --- a/cmd/fleetctl/get.go +++ b/cmd/fleetctl/get.go @@ -737,16 +737,22 @@ func getCarvesCommand() *cli.Command { completion = "Expired" } + errored := "no" + if c.Error != nil { + errored = "yes" + } + data = append(data, []string{ strconv.FormatInt(c.ID, 10), - c.CreatedAt.Local().String(), + c.CreatedAt.String(), c.RequestId, strconv.FormatInt(c.CarveSize, 10), completion, + errored, }) } - columns := []string{"id", "created_at", "request_id", "carve_size", "completion"} + columns := []string{"id", "created_at", "request_id", "carve_size", "completion", "errored"} printTable(c, columns, data) return nil @@ -789,6 +795,15 @@ func getCarveCommand() *cli.Command { return errors.New("-stdout and -outfile must not be specified together") } + carve, err := client.GetCarve(id) + if err != nil { + return err + } + + if carve.Error != nil { + return errors.New(*carve.Error) + } + if stdout || outFile != "" { out := os.Stdout if outFile != "" { @@ -812,11 +827,6 @@ func getCarveCommand() *cli.Command { return nil } - carve, err := client.GetCarve(id) - if err != nil { - return err - } - if err := printYaml(carve, c.App.Writer); err != nil { return fmt.Errorf("print carve yaml: %w", err) } diff --git a/cmd/fleetctl/get_test.go b/cmd/fleetctl/get_test.go index 27e6b56ac3..ccd0358204 100644 --- a/cmd/fleetctl/get_test.go +++ b/cmd/fleetctl/get_test.go @@ -1460,3 +1460,110 @@ func TestGetAppleMDM(t *testing.T) { expected := `Error: No Apple Push Notification service (APNs) certificate found.` assert.Contains(t, runAppForTest(t, []string{"get", "mdm_apple"}), expected) } + +func TestGetCarves(t *testing.T) { + _, ds := runServerWithMockedDS(t) + + createdAt, err := time.Parse(time.RFC3339, "1999-03-10T02:45:06.371Z") + require.NoError(t, err) + ds.ListCarvesFunc = func(ctx context.Context, opts fleet.CarveListOptions) ([]*fleet.CarveMetadata, error) { + return []*fleet.CarveMetadata{ + { + HostId: 1, + Name: "foobar", + BlockCount: 10, + BlockSize: 12, + CarveSize: 123, + CarveId: "carve_id_1", + RequestId: "request_id_1", + SessionId: "session_id_1", + CreatedAt: createdAt, + }, + { + HostId: 2, + Name: "barfoo", + BlockCount: 20, + BlockSize: 44, + CarveSize: 123, + CarveId: "carve_id_2", + RequestId: "request_id_2", + SessionId: "session_id_2", + CreatedAt: createdAt, + Error: ptr.String("test error"), + }, + }, nil + } + + expected := `+----+--------------------------------+--------------+------------+------------+---------+ +| ID | CREATED AT | REQUEST ID | CARVE SIZE | COMPLETION | ERRORED | ++----+--------------------------------+--------------+------------+------------+---------+ +| 0 | 1999-03-10 02:45:06.371 +0000 | request_id_1 | 123 | 10% | no | +| | UTC | | | | | ++----+--------------------------------+--------------+------------+------------+---------+ +| 0 | 1999-03-10 02:45:06.371 +0000 | request_id_2 | 123 | 5% | yes | +| | UTC | | | | | ++----+--------------------------------+--------------+------------+------------+---------+ +` + assert.Equal(t, expected, runAppForTest(t, []string{"get", "carves"})) +} + +func TestGetCarve(t *testing.T) { + _, ds := runServerWithMockedDS(t) + + createdAt, err := time.Parse(time.RFC3339, "1999-03-10T02:45:06.371Z") + require.NoError(t, err) + ds.CarveFunc = func(ctx context.Context, carveID int64) (*fleet.CarveMetadata, error) { + return &fleet.CarveMetadata{ + HostId: 1, + Name: "foobar", + BlockCount: 10, + BlockSize: 12, + CarveSize: 123, + CarveId: "carve_id_1", + RequestId: "request_id_1", + SessionId: "session_id_1", + CreatedAt: createdAt, + }, nil + } + + expectedOut := `--- +block_count: 10 +block_size: 12 +carve_id: carve_id_1 +carve_size: 123 +created_at: "1999-03-10T02:45:06.371Z" +error: null +expired: false +host_id: 1 +id: 0 +max_block: 0 +name: foobar +request_id: request_id_1 +session_id: session_id_1 +` + + assert.Equal(t, expectedOut, runAppForTest(t, []string{"get", "carve", "1"})) +} + +func TestGetCarveWithError(t *testing.T) { + _, ds := runServerWithMockedDS(t) + + createdAt, err := time.Parse(time.RFC3339, "1999-03-10T02:45:06.371Z") + require.NoError(t, err) + ds.CarveFunc = func(ctx context.Context, carveID int64) (*fleet.CarveMetadata, error) { + return &fleet.CarveMetadata{ + HostId: 1, + Name: "foobar", + BlockCount: 10, + BlockSize: 12, + CarveSize: 123, + CarveId: "carve_id_1", + RequestId: "request_id_1", + SessionId: "session_id_1", + CreatedAt: createdAt, + Error: ptr.String("test error"), + }, nil + } + + runAppCheckErr(t, []string{"get", "carve", "1"}, "test error") +} diff --git a/docs/Using-Fleet/REST-API.md b/docs/Using-Fleet/REST-API.md index 82c5cb8cec..f8decb6907 100644 --- a/docs/Using-Fleet/REST-API.md +++ b/docs/Using-Fleet/REST-API.md @@ -653,7 +653,8 @@ None. "request_id": "fleet_distributed_query_31", "session_id": "f73922ed-40a4-4e98-a50a-ccda9d3eb755", "expired": false, - "max_block": 1 + "max_block": 1, + "error": "S3 multipart carve upload: EntityTooSmall: Your proposed upload is smaller than the minimum allowed object size" } ] } diff --git a/server/datastore/mysql/carves.go b/server/datastore/mysql/carves.go index d88a63cf47..7980cc6d9e 100644 --- a/server/datastore/mysql/carves.go +++ b/server/datastore/mysql/carves.go @@ -11,7 +11,7 @@ import ( "github.com/jmoiron/sqlx" ) -func (ds *Datastore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata) (*fleet.CarveMetadata, error) { +func upsertCarveDB(ctx context.Context, writer sqlx.ExecerContext, metadata *fleet.CarveMetadata) (int64, error) { stmt := `INSERT INTO carve_metadata ( host_id, created_at, @@ -21,7 +21,8 @@ func (ds *Datastore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata carve_size, carve_id, request_id, - session_id + session_id, + error ) VALUES ( ?, ?, @@ -31,10 +32,11 @@ func (ds *Datastore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata ?, ?, ?, + ?, ? )` - result, err := ds.writer.ExecContext( + result, err := writer.ExecContext( ctx, stmt, metadata.HostId, @@ -46,14 +48,20 @@ func (ds *Datastore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata metadata.CarveId, metadata.RequestId, metadata.SessionId, + metadata.Error, ) + if err != nil { + return 0, ctxerr.Wrap(ctx, err, "insert carve metadata") + } + return result.LastInsertId() +} + +func (ds *Datastore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata) (*fleet.CarveMetadata, error) { + id, err := upsertCarveDB(ctx, ds.writer, metadata) if err != nil { return nil, ctxerr.Wrap(ctx, err, "insert carve metadata") } - - id, _ := result.LastInsertId() metadata.ID = id - return metadata, nil } @@ -67,7 +75,8 @@ func updateCarveDB(ctx context.Context, exec sqlx.ExecerContext, metadata *fleet stmt := ` UPDATE carve_metadata SET max_block = ?, - expired = ? + expired = ?, + error = ? WHERE id = ? ` _, err := exec.ExecContext( @@ -75,6 +84,7 @@ func updateCarveDB(ctx context.Context, exec sqlx.ExecerContext, metadata *fleet stmt, metadata.MaxBlock, metadata.Expired, + metadata.Error, metadata.ID, ) return ctxerr.Wrap(ctx, err, "update carve metadata") @@ -154,7 +164,8 @@ const carveSelectFields = ` request_id, session_id, expired, - max_block + max_block, + error ` func (ds *Datastore) Carve(ctx context.Context, carveId int64) (*fleet.CarveMetadata, error) { diff --git a/server/datastore/mysql/migrations/tables/20221205112142_AddErrorToCarveMetadata.go b/server/datastore/mysql/migrations/tables/20221205112142_AddErrorToCarveMetadata.go new file mode 100644 index 0000000000..b76feb1aa2 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20221205112142_AddErrorToCarveMetadata.go @@ -0,0 +1,20 @@ +package tables + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20221205112142, Down_20221205112142) +} + +func Up_20221205112142(tx *sql.Tx) error { + _, err := tx.Exec("ALTER TABLE `carve_metadata` ADD COLUMN `error` TEXT") + return errors.Wrap(err, "adding error column to carve_metadata") +} + +func Down_20221205112142(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20221205112142_AddErrorToCarveMetadata_test.go b/server/datastore/mysql/migrations/tables/20221205112142_AddErrorToCarveMetadata_test.go new file mode 100644 index 0000000000..569c06764e --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20221205112142_AddErrorToCarveMetadata_test.go @@ -0,0 +1,53 @@ +package tables + +import ( + "database/sql" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUp_20221205112142(t *testing.T) { + db := applyUpToPrev(t) + query := ` +INSERT INTO carve_metadata + (host_id, block_count, block_size, carve_size, carve_id, request_id, session_id) +VALUES + (1, 10, 1000, 10000, "carve_id", "request_id", ?) +` + + execNoErr(t, db, "INSERT INTO hosts (hostname, osquery_host_id) VALUES ('foo.example.com', 'foo')") + execNoErr(t, db, query, 1) + execNoErr(t, db, query, 2) + + // Apply current migration. + applyNext(t, db) + + // Okay if we don't provide an error + execNoErr(t, db, query, 3) + // Insert with an error + execNoErr(t, db, ` +INSERT INTO carve_metadata + (host_id, block_count, block_size, carve_size, carve_id, request_id, session_id, error) +VALUES + (1, 10, 1000, 10000, "carve_id", "request_id", 4, "made_up_error") +`) + // Update an existing row to add an error + execNoErr(t, db, `UPDATE carve_metadata SET error = "updated_error" WHERE session_id = 3`) + + var storedErr sql.NullString + row := db.QueryRow(`SELECT error FROM carve_metadata WHERE session_id = 3`) + err := row.Scan(&storedErr) + require.NoError(t, err) + require.Equal(t, "updated_error", storedErr.String) + + row = db.QueryRow(`SELECT error FROM carve_metadata WHERE session_id = 4`) + err = row.Scan(&storedErr) + require.NoError(t, err) + require.Equal(t, "made_up_error", storedErr.String) + + row = db.QueryRow(`SELECT error FROM carve_metadata WHERE session_id = 1`) + err = row.Scan(&storedErr) + require.NoError(t, err) + require.Equal(t, "", storedErr.String) +} diff --git a/server/datastore/mysql/migrations/tables/migration_test.go b/server/datastore/mysql/migrations/tables/migration_test.go index 2a4f08fb25..8de460812e 100644 --- a/server/datastore/mysql/migrations/tables/migration_test.go +++ b/server/datastore/mysql/migrations/tables/migration_test.go @@ -76,8 +76,8 @@ func applyUpToPrev(t *testing.T) *sqlx.DB { } } -func execNoErr(t *testing.T, db *sqlx.DB, query string) { - _, err := db.Exec(query) +func execNoErr(t *testing.T, db *sqlx.DB, query string, args ...any) { + _, err := db.Exec(query, args...) require.NoError(t, err) } diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index a8aecc55a2..ced2f394f4 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -62,6 +62,7 @@ CREATE TABLE `carve_metadata` ( `session_id` varchar(255) NOT NULL, `expired` tinyint(4) DEFAULT '0', `max_block` int(11) DEFAULT '-1', + `error` text, PRIMARY KEY (`id`), UNIQUE KEY `idx_session_id` (`session_id`), UNIQUE KEY `idx_name` (`name`), @@ -465,9 +466,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=160 DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB AUTO_INCREMENT=161 DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `mobile_device_management_solutions` ( diff --git a/server/datastore/s3/carves.go b/server/datastore/s3/carves.go index 110fbe92d8..8af65706db 100644 --- a/server/datastore/s3/carves.go +++ b/server/datastore/s3/carves.go @@ -14,6 +14,7 @@ import ( "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" ) const ( @@ -56,11 +57,24 @@ func (c *CarveStore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata Bucket: &c.bucket, Key: &objectKey, }) + if err != nil { - return nil, ctxerr.Wrap(ctx, err, "s3 multipart carve create") + // even if we fail to create the multipart upload, we still want to create + // the carve in the database and register an error, this way the user can + // still fetch the carve and check its status + metadata.Error = ptr.String(err.Error()) + if _, err := c.metadatadb.NewCarve(ctx, metadata); err != nil { + return nil, ctxerr.Wrap(ctx, err, "creating carve metadata") + } + return nil, ctxerr.Wrap(ctx, err, "creating multipart upload") } + metadata.SessionId = *res.UploadId - return c.metadatadb.NewCarve(ctx, metadata) + savedMetadata, err := c.metadatadb.NewCarve(ctx, metadata) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "creating carve metadata") + } + return savedMetadata, nil } // UpdateCarve updates carve definition in database diff --git a/server/datastore/s3/carves_test.go b/server/datastore/s3/carves_test.go new file mode 100644 index 0000000000..3ed7f97237 --- /dev/null +++ b/server/datastore/s3/carves_test.go @@ -0,0 +1 @@ +package s3 diff --git a/server/fleet/carves.go b/server/fleet/carves.go index 418b09cadc..e67081886b 100644 --- a/server/fleet/carves.go +++ b/server/fleet/carves.go @@ -27,6 +27,8 @@ type CarveMetadata struct { SessionId string `json:"session_id" db:"session_id"` // Expired is whether the carve has "expired" (data has been purged). Expired bool `json:"expired" db:"expired"` + // Error is the error message if the carve failed. + Error *string `json:"error" db:"error"` // MaxBlock is the highest block number currently stored for this carve. // This value is not stored directly, but generated from the carve_blocks diff --git a/server/service/carves.go b/server/service/carves.go index 9c299102e8..af592ac635 100644 --- a/server/service/carves.go +++ b/server/service/carves.go @@ -8,7 +8,9 @@ import ( "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" hostctx "github.com/fleetdm/fleet/v4/server/contexts/host" + "github.com/fleetdm/fleet/v4/server/contexts/logging" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" "github.com/google/uuid" ) @@ -290,6 +292,28 @@ func (svc *Service) CarveBlock(ctx context.Context, payload fleet.CarveBlockPayl // Request is now authenticated + if err := svc.validateCarveBlock(payload, carve); err != nil { + carve.Error = ptr.String(err.Error()) + if errRecord := svc.carveStore.UpdateCarve(ctx, carve); err != nil { + logging.WithExtras(ctx, "validate_carve_error", errRecord, "carve_id", carve.ID) + } + + return ctxerr.Wrap(ctx, err, "validate carve block") + } + + if err := svc.carveStore.NewBlock(ctx, carve, payload.BlockId, payload.Data); err != nil { + carve.Error = ptr.String(err.Error()) + if errRecord := svc.carveStore.UpdateCarve(ctx, carve); err != nil { + logging.WithExtras(ctx, "record_carve_error", errRecord, "carve_id", carve.ID) + } + + return ctxerr.Wrap(ctx, err, "save carve block data") + } + + return nil +} + +func (svc *Service) validateCarveBlock(payload fleet.CarveBlockPayload, carve *fleet.CarveMetadata) error { if payload.BlockId > carve.BlockCount-1 { return fmt.Errorf("block_id exceeds expected max (%d): %d", carve.BlockCount-1, payload.BlockId) } @@ -302,9 +326,5 @@ func (svc *Service) CarveBlock(ctx context.Context, payload fleet.CarveBlockPayl return fmt.Errorf("exceeded declared block size %d: %d", carve.BlockSize, len(payload.Data)) } - if err := svc.carveStore.NewBlock(ctx, carve, payload.BlockId, payload.Data); err != nil { - return ctxerr.Wrap(ctx, err, "save block data") - } - return nil } diff --git a/server/service/carves_test.go b/server/service/carves_test.go index 249873f3f2..d338fd4399 100644 --- a/server/service/carves_test.go +++ b/server/service/carves_test.go @@ -459,6 +459,11 @@ func TestCarveCarveBlockBlockCountExceedError(t *testing.T) { assert.Equal(t, metadata.SessionId, sessionId) return metadata, nil } + ms.UpdateCarveFunc = func(ctx context.Context, carve *fleet.CarveMetadata) error { + assert.NotNil(t, carve.Error) + assert.Equal(t, *carve.Error, "block_id exceeds expected max (22): 23") + return nil + } payload := fleet.CarveBlockPayload{ Data: []byte("this is the carve data :)"), @@ -490,6 +495,11 @@ func TestCarveCarveBlockBlockCountMatchError(t *testing.T) { assert.Equal(t, metadata.SessionId, sessionId) return metadata, nil } + ms.UpdateCarveFunc = func(ctx context.Context, carve *fleet.CarveMetadata) error { + assert.NotNil(t, carve.Error) + assert.Equal(t, *carve.Error, "block_id does not match expected block (4): 7") + return nil + } payload := fleet.CarveBlockPayload{ Data: []byte("this is the carve data :)"), @@ -521,6 +531,11 @@ func TestCarveCarveBlockBlockSizeError(t *testing.T) { assert.Equal(t, metadata.SessionId, sessionId) return metadata, nil } + ms.UpdateCarveFunc = func(ctx context.Context, carve *fleet.CarveMetadata) error { + assert.NotNil(t, carve.Error) + assert.Equal(t, *carve.Error, "exceeded declared block size 16: 37") + return nil + } payload := fleet.CarveBlockPayload{ Data: []byte("this is the carve data :) TOO LONG!!!"), @@ -555,6 +570,11 @@ func TestCarveCarveBlockNewBlockError(t *testing.T) { ms.NewBlockFunc = func(ctx context.Context, carve *fleet.CarveMetadata, blockId int64, data []byte) error { return errors.New("kaboom!") } + ms.UpdateCarveFunc = func(ctx context.Context, carve *fleet.CarveMetadata) error { + assert.NotNil(t, carve.Error) + assert.Equal(t, *carve.Error, "kaboom!") + return nil + } payload := fleet.CarveBlockPayload{ Data: []byte("this is the carve data :)"), diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index a4a6142464..15928b5a59 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -5195,6 +5195,12 @@ func (s *integrationTestSuite) TestCarve() { Data: []byte("p1."), }, http.StatusInternalServerError, &blockResp) // TODO: should be 400, see #4406 + checkCarveError := func(id uint, err string) { + var getResp getCarveResponse + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/carves/%d", id), nil, http.StatusOK, &getResp) + require.Equal(t, err, *getResp.Carve.Error) + } + // sending a block with unexpected block id (expects 0, got 1) s.DoJSON("POST", "/api/osquery/carve/block", carveBlockRequest{ BlockId: 1, @@ -5202,6 +5208,7 @@ func (s *integrationTestSuite) TestCarve() { RequestId: "r1", Data: []byte("p1."), }, http.StatusInternalServerError, &blockResp) // TODO: should be 400, see #4406 + checkCarveError(1, "block_id does not match expected block (0): 1") // sending a block with valid payload, block 0 s.DoJSON("POST", "/api/osquery/carve/block", carveBlockRequest{ @@ -5230,6 +5237,7 @@ func (s *integrationTestSuite) TestCarve() { RequestId: "r1", Data: []byte("p2."), }, http.StatusInternalServerError, &blockResp) // TODO: should be 400, see #4406 + checkCarveError(1, "block_id does not match expected block (2): 1") // sending final block with too many bytes blockResp = carveBlockResponse{} @@ -5239,6 +5247,7 @@ func (s *integrationTestSuite) TestCarve() { RequestId: "r1", Data: []byte("p3extra"), }, http.StatusInternalServerError, &blockResp) // TODO: should be 400, see #4406 + checkCarveError(1, "exceeded declared block size 3: 7") // sending actual final block blockResp = carveBlockResponse{} @@ -5258,6 +5267,7 @@ func (s *integrationTestSuite) TestCarve() { RequestId: "r1", Data: []byte("p4."), }, http.StatusInternalServerError, &blockResp) // TODO: should be 400, see #4406 + checkCarveError(1, "block_id exceeds expected max (2): 3") } func (s *integrationTestSuite) TestPasswordReset() {