Add fleetctl upgrade-packs command to migrate 2017 packs to queries (#13078)

This commit is contained in:
Martin Angers
2023-08-08 08:21:57 -04:00
committed by GitHub
parent 20e3c9dc4f
commit 37ba43d404
8 changed files with 786 additions and 16 deletions
+1
View File
@@ -0,0 +1 @@
* Added the `fleetctl upgrade-packs` command to migrate 2017 packs to the new combined schedule and query concept.
+1
View File
@@ -106,6 +106,7 @@ func createApp(
},
triggerCommand(),
mdmCommand(),
upgradePacksCommand(),
}
return app
}
+11 -4
View File
@@ -512,7 +512,7 @@ func getPacksCommand() *cli.Command {
return &cli.Command{
Name: "packs",
Aliases: []string{"pack", "p"},
Usage: "List information about one or more packs",
Usage: `Retrieve 2017 "Packs" data for migration into modern osquery packs`,
Flags: []cli.Flag{
&cli.BoolFlag{
Name: withQueriesFlagName,
@@ -574,7 +574,7 @@ func getPacksCommand() *cli.Command {
// if name wasn't provided, list all packs
if name == "" {
packs, err := client.GetPacks()
packs, err := client.GetPacksSpecs()
if err != nil {
return fmt.Errorf("could not list packs: %w", err)
}
@@ -590,7 +590,7 @@ func getPacksCommand() *cli.Command {
}
if len(packs) == 0 {
fmt.Println("No packs found")
log(c, "No 2017 \"Packs\" found.\n")
return nil
}
@@ -607,12 +607,19 @@ func getPacksCommand() *cli.Command {
columns := []string{"name", "platform", "description", "disabled"}
printTable(c, columns, data)
log(c, fmt.Sprintf(`Found %d 2017 "Packs".
Querying in Fleet is becoming more powerful. To learn more, visit:
https://fleetdm.com/handbook/company/why-this-way#why-does-fleet-support-query-packs
To retrieve "Pack" data in a portable format for upgrading, run `+"`fleetctl upgrade-packs`"+`.
`, len(packs)))
return nil
}
// Name was specified
pack, err := client.GetPack(name)
pack, err := client.GetPackSpec(name)
if err != nil {
return err
}
+34
View File
@@ -871,6 +871,12 @@ func TestGetPacks(t *testing.T) {
+-------+----------+-------------+----------+
| pack1 | darwin | some desc | false |
+-------+----------+-------------+----------+
Found 1 2017 "Packs".
Querying in Fleet is becoming more powerful. To learn more, visit:
https://fleetdm.com/handbook/company/why-this-way#why-does-fleet-support-query-packs
To retrieve "Pack" data in a portable format for upgrading, run ` + "`fleetctl upgrade-packs`" + `.
`
expectedYaml := `---
apiVersion: v1
@@ -906,6 +912,17 @@ spec:
assert.Equal(t, expected, runAppForTest(t, []string{"get", "packs"}))
assert.YAMLEq(t, expectedYaml, runAppForTest(t, []string{"get", "packs", "--yaml"}))
assert.JSONEq(t, expectedJson, runAppForTest(t, []string{"get", "packs", "--json"}))
// test output when there are no packs
ds.GetPackSpecsFunc = func(ctx context.Context) ([]*fleet.PackSpec, error) {
return nil, nil
}
expected = `No 2017 "Packs" found.
`
assert.Equal(t, expected, runAppForTest(t, []string{"get", "packs"}))
assert.Empty(t, runAppForTest(t, []string{"get", "packs", "--yaml"}))
assert.Empty(t, runAppForTest(t, []string{"get", "packs", "--json"}))
}
func TestGetPack(t *testing.T) {
@@ -970,6 +987,23 @@ spec:
assert.YAMLEq(t, expectedYaml, runAppForTest(t, []string{"get", "packs", "pack1"}))
assert.YAMLEq(t, expectedYaml, runAppForTest(t, []string{"get", "packs", "--yaml", "pack1"}))
assert.JSONEq(t, expectedJson, runAppForTest(t, []string{"get", "packs", "--json", "pack1"}))
expectedEmptyYaml := `---
apiVersion: v1
kind: pack
spec: null
`
expectedEmptyJson := `
{
"kind": "pack",
"apiVersion": "v1",
"spec": null
}`
assert.YAMLEq(t, expectedEmptyYaml, runAppForTest(t, []string{"get", "packs", "no-such-pack"}))
assert.YAMLEq(t, expectedEmptyYaml, runAppForTest(t, []string{"get", "packs", "--yaml", "no-such-pack"}))
assert.JSONEq(t, expectedEmptyJson, runAppForTest(t, []string{"get", "packs", "--json", "no-such-pack"}))
}
func TestGetQueries(t *testing.T) {
+275
View File
@@ -0,0 +1,275 @@
package main
import (
"errors"
"fmt"
"os"
"strings"
"time"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/ghodss/yaml"
"github.com/urfave/cli/v2"
)
// variable used by tests to set a predictable timestamp
var testUpgradePacksTimestamp time.Time
func upgradePacksCommand() *cli.Command {
var outputFilename string
return &cli.Command{
Name: "upgrade-packs",
Usage: `Generate a config file to assist with converting 2017 "Packs" into portable queries that run on a schedule`,
UsageText: `fleetctl upgrade-packs [options]`,
Flags: []cli.Flag{
configFlag(),
contextFlag(),
debugFlag(),
&cli.StringFlag{
Name: "o",
EnvVars: []string{"OUTPUT_FILENAME"},
Value: "",
Destination: &outputFilename,
Usage: "The name of the file to output converted results",
Required: true,
},
},
Action: func(c *cli.Context) error {
client, err := clientFromCLI(c)
if err != nil {
return err
}
// must be an admin, but reading packs and queries does not require being an admin,
// so this must be validated separately, before loading the packs.
user, err := client.Me()
if err != nil {
return fmt.Errorf("check user role: %w", err)
}
if user.GlobalRole == nil || *user.GlobalRole != fleet.RoleAdmin {
return errors.New("could not upgrade packs: forbidden: user does not have the admin role")
}
// read the server settings so that the packs URL can be printed in the output
appCfg, err := client.GetAppConfig()
if err != nil {
return fmt.Errorf("could not read configuration: %w", err)
}
// read the packs and queries
packsSpecs, err := client.GetPacksSpecs()
if err != nil {
return fmt.Errorf("could not list packs specs: %w", err)
}
if len(packsSpecs) == 0 {
log(c, "No 2017 \"Packs\" found.\n")
return nil
}
// must read the DB packs too (not just the specs) in order to get the
// host targets of the packs, which are not retrieved by GetPacksSpecs
// (because host targets cannot be set via the apply spec).
packsDB, err := client.ListPacks()
if err != nil {
return fmt.Errorf("could not list packs: %w", err)
}
// map the DB packs by ID
packsByID := make(map[uint]*fleet.Pack, len(packsDB))
for _, p := range packsDB {
packsByID[p.ID] = p
}
// get global queries (teamID==nil), because 2017 packs reference global queries.
queries, err := client.GetQueries(nil)
if err != nil {
return fmt.Errorf("could not list queries: %w", err)
}
// map queries by packs that reference them
queriesByPack := mapQueriesToPacks(packsSpecs, queries)
var (
newSpecs []*fleet.QuerySpec
convertedQueries int
)
// use a consistent upgrade timestamp for all new queries (used to make name unique)
upgradeTimestamp := testUpgradePacksTimestamp
if upgradeTimestamp.IsZero() {
upgradeTimestamp = time.Now()
}
for _, packSpec := range packsSpecs {
newPackSpecs, convPackQueries := upgradePackToQueriesSpecs(packSpec, packsByID[packSpec.ID], queriesByPack[packSpec], upgradeTimestamp)
newSpecs = append(newSpecs, newPackSpecs...)
convertedQueries += convPackQueries
}
if err := writeQuerySpecsToFile(outputFilename, newSpecs); err != nil {
return fmt.Errorf("could not write queries to file: %w", err)
}
log(c, fmt.Sprintf(`Converted %d queries from %d 2017 "Packs" into portable queries:
For any "Packs" targeting teams, duplicate queries were written for each team.
For any "Packs" targeting labels or individual hosts, a global query was written without scheduling features enabled.
To import these queries to Fleet, you can merge the data in the output file with your existing query configuration and run `+"`fleetctl apply`"+`.
Note that existing 2017 "Packs" have been left intact. To avoid running duplicate queries on your hosts, visit %s/packs/manage and disable all 2017 "Packs" after upgrading. Fleet will continue to support these until the next major version release, when 2017 "Packs" will be automatically converted to queries.
`, convertedQueries, len(packsSpecs), appCfg.ServerSettings.ServerURL))
return nil
},
}
}
func writeQuerySpecsToFile(filename string, specs []*fleet.QuerySpec) error {
f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, defaultFileMode)
if err != nil {
return err
}
defer f.Close()
for _, spec := range specs {
qYaml := fleet.QueryObject{
ObjectMetadata: fleet.ObjectMetadata{
ApiVersion: fleet.ApiVersion,
Kind: fleet.QueryKind,
},
Spec: *spec,
}
yml, err := yaml.Marshal(qYaml)
if err != nil {
return err
}
if _, err := fmt.Fprint(f, string(yml)+"---\n"); err != nil {
return err
}
}
return f.Close()
}
func mapQueriesToPacks(packs []*fleet.PackSpec, queries []fleet.Query) map[*fleet.PackSpec][]*fleet.Query {
queriesByName := make(map[string]*fleet.Query, len(queries))
for _, q := range queries {
q := q // avoid taking address of iteration var
queriesByName[q.Name] = &q
}
queriesByPack := make(map[*fleet.PackSpec][]*fleet.Query, len(packs))
for _, pack := range packs {
for _, sq := range pack.Queries {
if q := queriesByName[sq.QueryName]; q != nil {
queriesByPack[pack] = append(queriesByPack[pack], queriesByName[sq.QueryName])
}
}
}
return queriesByPack
}
// upgrades the pack to the new query format, duplicating queries as needed
// (pack queries targeting teams are duplicated for each team, pack queries
// targeting labels or hosts are duplicated as a global query). Returns the
// generated new query specs and the number of pack queries that were
// converted.
func upgradePackToQueriesSpecs(packSpec *fleet.PackSpec, packDB *fleet.Pack, packQueries []*fleet.Query, ts time.Time) ([]*fleet.QuerySpec, int) {
if len(packQueries) == 0 {
// if the pack has no query, there's nothing to convert
return nil, 0
}
var targetsHosts bool
if packDB != nil {
targetsHosts = len(packDB.Hosts) > 0
}
schedByName := make(map[string]*fleet.PackSpecQuery, len(packSpec.Queries))
for _, sq := range packSpec.Queries {
sq := sq // avoid taking the address of iteration var
schedByName[sq.QueryName] = &sq
}
var (
newSpecs []*fleet.QuerySpec
convertedQueries int
)
for _, pq := range packQueries {
sched := schedByName[pq.Name]
if sched == nil {
continue
}
desc := pq.Description
if desc != "" && !strings.HasSuffix(desc, "\n") {
desc += "\n"
}
desc += fmt.Sprintf("(converted from pack %q, query %q)", packSpec.Name, pq.Name)
var loggingType string
if sched.Snapshot != nil && *sched.Snapshot {
loggingType = "snapshot"
} else if sched.Removed != nil {
if *sched.Removed {
loggingType = "differential"
} else {
loggingType = "differential_ignore_removals"
}
}
var (
schedPlatform string
schedVersion string
converted bool
)
if sched.Platform != nil {
schedPlatform = *sched.Platform
}
if sched.Version != nil {
schedVersion = *sched.Version
}
for _, tm := range packSpec.Targets.Teams {
converted = true
// duplicate the query for each targeted team
newQueryName := fmt.Sprintf("%s - %s - %s - %s", packSpec.Name, pq.Name, tm, ts.Format("Jan _2 15:04:05.000"))
newSpecs = append(newSpecs, &fleet.QuerySpec{
Name: newQueryName,
Description: desc,
Query: pq.Query,
TeamName: tm,
Interval: sched.Interval,
ObserverCanRun: pq.ObserverCanRun,
Platform: schedPlatform,
MinOsqueryVersion: schedVersion,
AutomationsEnabled: !packSpec.Disabled,
Logging: loggingType,
})
}
if len(packSpec.Targets.Labels) > 0 || targetsHosts {
converted = true
// write a global query without scheduling features
newQueryName := fmt.Sprintf("%s - %s - %s", packSpec.Name, pq.Name, ts.Format("Jan _2 15:04:05.000"))
newSpecs = append(newSpecs, &fleet.QuerySpec{
Name: newQueryName,
Description: desc,
Query: pq.Query,
TeamName: "",
Interval: 0,
ObserverCanRun: pq.ObserverCanRun,
Platform: schedPlatform,
MinOsqueryVersion: schedVersion,
AutomationsEnabled: false,
Logging: loggingType,
})
}
if converted {
convertedQueries++
}
}
return newSpecs, convertedQueries
}
+434
View File
@@ -0,0 +1,434 @@
package main
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/ptr"
"github.com/stretchr/testify/require"
)
func TestUpgradeSinglePack(t *testing.T) {
ts := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
cases := []struct {
desc string
pack *fleet.Pack
queries []*fleet.Query
scheds []fleet.PackSpecQuery
want []*fleet.QuerySpec
wantCount int
}{
{
desc: "no queries, a target",
pack: &fleet.Pack{Name: "p1", Teams: []fleet.Target{{Type: fleet.TargetTeam, DisplayText: "t1"}}},
queries: nil,
scheds: nil,
want: nil,
wantCount: 0,
},
{
desc: "no queries, no target",
pack: &fleet.Pack{Name: "p1"},
queries: nil,
scheds: nil,
want: nil,
wantCount: 0,
},
{
desc: "a query, no target",
pack: &fleet.Pack{Name: "p1"},
queries: []*fleet.Query{{Name: "q1", Query: "select 1"}},
scheds: []fleet.PackSpecQuery{{QueryName: "q1", Interval: 60}},
want: nil,
wantCount: 0,
},
{
desc: "a query, label target",
pack: &fleet.Pack{Name: "p1", Labels: []fleet.Target{{Type: fleet.TargetLabel, DisplayText: "l1"}}},
queries: []*fleet.Query{{Name: "q1", Query: "select 1"}},
scheds: []fleet.PackSpecQuery{{QueryName: "q1", Interval: 60}},
want: []*fleet.QuerySpec{
// global query, schedule is removed
{Name: "p1 - q1 - Jan 1 00:00:00.000", Description: `(converted from pack "p1", query "q1")`, Query: "select 1", Interval: 0},
},
wantCount: 1,
},
{
desc: "2 queries, host target",
pack: &fleet.Pack{Name: "p1", Hosts: []fleet.Target{{Type: fleet.TargetHost, DisplayText: "h1"}}},
queries: []*fleet.Query{
{Name: "q1", Query: "select 1"},
{Name: "q2", Query: "select 2", ObserverCanRun: true, Description: "q2 desc"},
},
scheds: []fleet.PackSpecQuery{
{QueryName: "q1", Interval: 60, Name: "sq1", Snapshot: ptr.Bool(true), Platform: ptr.String("darwin"), Version: ptr.String("v1")},
{QueryName: "q2", Interval: 90, Name: "sq2", Description: "sq2 desc"},
},
want: []*fleet.QuerySpec{
// global queries, schedule is removed
{Name: "p1 - q1 - Jan 1 00:00:00.000", Description: `(converted from pack "p1", query "q1")`, Query: "select 1", Interval: 0, Logging: "snapshot", Platform: "darwin", MinOsqueryVersion: "v1"},
{Name: "p1 - q2 - Jan 1 00:00:00.000", Description: "q2 desc\n(converted from pack \"p1\", query \"q2\")", Query: "select 2", Interval: 0, ObserverCanRun: true},
},
wantCount: 2,
},
{
desc: "2 queries, 2 team targets",
pack: &fleet.Pack{Name: "p1", Description: "p1 desc", Platform: "ignored", Teams: []fleet.Target{
{Type: fleet.TargetTeam, DisplayText: "t1"},
{Type: fleet.TargetTeam, DisplayText: "t2"},
}},
queries: []*fleet.Query{
{Name: "q1", Query: "select 1"},
{Name: "q2", Query: "select 2", ObserverCanRun: true, Description: "q2 desc"},
},
scheds: []fleet.PackSpecQuery{
{QueryName: "q1", Interval: 60, Name: "sq1", Snapshot: ptr.Bool(true), Removed: ptr.Bool(true), Platform: ptr.String("darwin"), Version: ptr.String("v1")},
{QueryName: "q2", Interval: 90, Name: "sq2", Removed: ptr.Bool(false), Description: "sq2 desc"},
},
want: []*fleet.QuerySpec{
// per-team queries
{Name: "p1 - q1 - t1 - Jan 1 00:00:00.000", Description: `(converted from pack "p1", query "q1")`, TeamName: "t1", AutomationsEnabled: true, Query: "select 1", Interval: 60, Logging: "snapshot", Platform: "darwin", MinOsqueryVersion: "v1"},
{Name: "p1 - q1 - t2 - Jan 1 00:00:00.000", Description: `(converted from pack "p1", query "q1")`, TeamName: "t2", AutomationsEnabled: true, Query: "select 1", Interval: 60, Logging: "snapshot", Platform: "darwin", MinOsqueryVersion: "v1"},
{Name: "p1 - q2 - t1 - Jan 1 00:00:00.000", Description: "q2 desc\n(converted from pack \"p1\", query \"q2\")", TeamName: "t1", AutomationsEnabled: true, Query: "select 2", Interval: 90, ObserverCanRun: true, Logging: "differential_ignore_removals"},
{Name: "p1 - q2 - t2 - Jan 1 00:00:00.000", Description: "q2 desc\n(converted from pack \"p1\", query \"q2\")", TeamName: "t2", AutomationsEnabled: true, Query: "select 2", Interval: 90, ObserverCanRun: true, Logging: "differential_ignore_removals"},
},
wantCount: 2,
},
{
desc: "2 queries, 2 team targets, label target",
pack: &fleet.Pack{Name: "p1", Description: "p1 desc", Platform: "ignored", Teams: []fleet.Target{
{Type: fleet.TargetTeam, DisplayText: "t1"},
{Type: fleet.TargetTeam, DisplayText: "t2"},
}, Labels: []fleet.Target{
{Type: fleet.TargetLabel, DisplayText: "l1"},
}},
queries: []*fleet.Query{
{Name: "q1", Query: "select 1"},
{Name: "q2", Query: "select 2", ObserverCanRun: true, Description: "q2 desc"},
},
scheds: []fleet.PackSpecQuery{
{QueryName: "q1", Interval: 60, Name: "sq1", Snapshot: ptr.Bool(true), Removed: ptr.Bool(true), Platform: ptr.String("darwin"), Version: ptr.String("v1")},
{QueryName: "q2", Interval: 90, Name: "sq2", Removed: ptr.Bool(false), Description: "sq2 desc"},
},
want: []*fleet.QuerySpec{
// per-team queries, and global queries with schedules removed
{Name: "p1 - q1 - t1 - Jan 1 00:00:00.000", Description: `(converted from pack "p1", query "q1")`, TeamName: "t1", AutomationsEnabled: true, Query: "select 1", Interval: 60, Logging: "snapshot", Platform: "darwin", MinOsqueryVersion: "v1"},
{Name: "p1 - q1 - t2 - Jan 1 00:00:00.000", Description: `(converted from pack "p1", query "q1")`, TeamName: "t2", AutomationsEnabled: true, Query: "select 1", Interval: 60, Logging: "snapshot", Platform: "darwin", MinOsqueryVersion: "v1"},
{Name: "p1 - q1 - Jan 1 00:00:00.000", Description: `(converted from pack "p1", query "q1")`, Query: "select 1", Interval: 0, Logging: "snapshot", Platform: "darwin", MinOsqueryVersion: "v1"},
{Name: "p1 - q2 - t1 - Jan 1 00:00:00.000", Description: "q2 desc\n(converted from pack \"p1\", query \"q2\")", TeamName: "t1", AutomationsEnabled: true, Query: "select 2", Interval: 90, ObserverCanRun: true, Logging: "differential_ignore_removals"},
{Name: "p1 - q2 - t2 - Jan 1 00:00:00.000", Description: "q2 desc\n(converted from pack \"p1\", query \"q2\")", TeamName: "t2", AutomationsEnabled: true, Query: "select 2", Interval: 90, ObserverCanRun: true, Logging: "differential_ignore_removals"},
{Name: "p1 - q2 - Jan 1 00:00:00.000", Description: "q2 desc\n(converted from pack \"p1\", query \"q2\")", Query: "select 2", Interval: 0, ObserverCanRun: true, Logging: "differential_ignore_removals"},
},
wantCount: 2,
},
{
desc: "2 queries, team target, host target",
pack: &fleet.Pack{Name: "p1", Description: "p1 desc", Platform: "ignored", Teams: []fleet.Target{
{Type: fleet.TargetTeam, DisplayText: "t1"},
}, Hosts: []fleet.Target{
{Type: fleet.TargetHost, DisplayText: "h1"},
}},
queries: []*fleet.Query{
{Name: "q1", Query: "select 1"},
{Name: "q2", Query: "select 2", ObserverCanRun: true, Description: "q2 desc"},
},
scheds: []fleet.PackSpecQuery{
{QueryName: "q1", Interval: 60, Name: "sq1", Removed: ptr.Bool(true), Platform: ptr.String("darwin"), Version: ptr.String("v1")},
{QueryName: "q2", Interval: 90, Name: "sq2", Removed: ptr.Bool(false), Description: "sq2 desc"},
},
want: []*fleet.QuerySpec{
// per-team queries, and global queries with schedules removed
{Name: "p1 - q1 - t1 - Jan 1 00:00:00.000", Description: `(converted from pack "p1", query "q1")`, TeamName: "t1", AutomationsEnabled: true, Query: "select 1", Interval: 60, Logging: "differential", Platform: "darwin", MinOsqueryVersion: "v1"},
{Name: "p1 - q1 - Jan 1 00:00:00.000", Description: `(converted from pack "p1", query "q1")`, Query: "select 1", Interval: 0, Logging: "differential", Platform: "darwin", MinOsqueryVersion: "v1"},
{Name: "p1 - q2 - t1 - Jan 1 00:00:00.000", Description: "q2 desc\n(converted from pack \"p1\", query \"q2\")", TeamName: "t1", AutomationsEnabled: true, Query: "select 2", Interval: 90, ObserverCanRun: true, Logging: "differential_ignore_removals"},
{Name: "p1 - q2 - Jan 1 00:00:00.000", Description: "q2 desc\n(converted from pack \"p1\", query \"q2\")", Query: "select 2", Interval: 0, ObserverCanRun: true, Logging: "differential_ignore_removals"},
},
wantCount: 2,
},
{
desc: "2 queries, all targets, a query with no schedule match",
pack: &fleet.Pack{Name: "p1", Description: "p1 desc", Platform: "ignored", Teams: []fleet.Target{
{Type: fleet.TargetTeam, DisplayText: "t1"},
}, Hosts: []fleet.Target{
{Type: fleet.TargetHost, DisplayText: "h1"},
}, Labels: []fleet.Target{
{Type: fleet.TargetLabel, DisplayText: "l1"},
}},
queries: []*fleet.Query{
{Name: "q1", Query: "select 1"},
{Name: "q2", Query: "select 2", ObserverCanRun: true, Description: "q2 desc"},
},
scheds: []fleet.PackSpecQuery{
{QueryName: "q1", Interval: 60, Name: "sq1", Removed: ptr.Bool(true), Platform: ptr.String("darwin"), Version: ptr.String("v1")},
{QueryName: "no-such-query", Interval: 90, Name: "sq2", Removed: ptr.Bool(false), Description: "sq2 desc"},
},
want: []*fleet.QuerySpec{
// per-team queries, and global queries with schedules removed
{Name: "p1 - q1 - t1 - Jan 1 00:00:00.000", Description: `(converted from pack "p1", query "q1")`, TeamName: "t1", AutomationsEnabled: true, Query: "select 1", Interval: 60, Logging: "differential", Platform: "darwin", MinOsqueryVersion: "v1"},
{Name: "p1 - q1 - Jan 1 00:00:00.000", Description: `(converted from pack "p1", query "q1")`, Query: "select 1", Interval: 0, Logging: "differential", Platform: "darwin", MinOsqueryVersion: "v1"},
},
wantCount: 1,
},
{
desc: "a query, team target, disabled pack",
pack: &fleet.Pack{Name: "p1", Disabled: true, Teams: []fleet.Target{{Type: fleet.TargetTeam, DisplayText: "t1"}}},
queries: []*fleet.Query{{Name: "q1", Query: "select 1"}},
scheds: []fleet.PackSpecQuery{{QueryName: "q1", Interval: 60}},
want: []*fleet.QuerySpec{
{Name: "p1 - q1 - t1 - Jan 1 00:00:00.000", Description: `(converted from pack "p1", query "q1")`, Query: "select 1", TeamName: "t1", AutomationsEnabled: false, Interval: 60},
},
wantCount: 1,
},
}
for _, c := range cases {
t.Run(c.desc, func(t *testing.T) {
// create the pack spec corresponding to the DB pack of the case
packSpec := &fleet.PackSpec{
Name: c.pack.Name,
Description: c.pack.Description,
Platform: c.pack.Platform,
Disabled: c.pack.Disabled,
Queries: c.scheds,
}
for _, tt := range c.pack.Teams {
packSpec.Targets.Teams = append(packSpec.Targets.Teams, tt.DisplayText)
}
for _, lt := range c.pack.Labels {
packSpec.Targets.Labels = append(packSpec.Targets.Labels, lt.DisplayText)
}
got, n := upgradePackToQueriesSpecs(packSpec, c.pack, c.queries, ts)
// Equal gives a better diff than ElementsMatch, so for maintainability of the
// test, it's worth it to keep the expected results in the same order as the
// actual ones.
require.Equal(t, c.want, got)
require.Equal(t, c.wantCount, n)
})
}
}
func TestFleetctlUpgradePacks_EmptyPacks(t *testing.T) {
_, ds := runServerWithMockedDS(t)
ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
return &fleet.AppConfig{ServerSettings: fleet.ServerSettings{ServerURL: "https://example.com"}}, nil
}
ds.UserByIDFunc = func(ctx context.Context, id uint) (*fleet.User, error) {
return &fleet.User{ID: id, GlobalRole: ptr.String(fleet.RoleAdmin)}, nil
}
ds.GetPackSpecsFunc = func(ctx context.Context) ([]*fleet.PackSpec, error) {
return []*fleet.PackSpec{
{Name: "p1", Targets: fleet.PackSpecTargets{Labels: []string{"l1"}}},
{Name: "p2", Targets: fleet.PackSpecTargets{Teams: []string{"t1"}}},
}, nil
}
ds.ListPacksFunc = func(ctx context.Context, opt fleet.PackListOptions) ([]*fleet.Pack, error) {
return []*fleet.Pack{
{Name: "p1", Labels: []fleet.Target{{Type: fleet.TargetLabel, DisplayText: "l1"}}, LabelIDs: []uint{1}},
{Name: "p2", Teams: []fleet.Target{{Type: fleet.TargetTeam, DisplayText: "t1"}}, TeamIDs: []uint{1}},
}, nil
}
ds.ListScheduledQueriesInPackWithStatsFunc = func(ctx context.Context, id uint, opts fleet.ListOptions) ([]*fleet.ScheduledQuery, error) {
return nil, nil
}
ds.CountHostsInTargetsFunc = func(ctx context.Context, filter fleet.TeamFilter, targets fleet.HostTargets, now time.Time) (fleet.TargetMetrics, error) {
return fleet.TargetMetrics{}, nil
}
ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) {
return nil, nil
}
tempDir := t.TempDir()
outputFile := filepath.Join(tempDir, "output.yml")
// write some dummy data in the file, it should be overwritten
err := os.WriteFile(outputFile, []byte("dummy"), 0644)
require.NoError(t, err)
got := runAppForTest(t, []string{"upgrade-packs", "-o", outputFile})
require.Contains(t, got, `Converted 0 queries from 2 2017 "Packs" into portable queries:`)
require.Contains(t, got, `visit https://example.com/packs/manage and disable all`)
content, err := os.ReadFile(outputFile)
require.NoError(t, err)
require.Empty(t, content)
}
func TestFleetctlUpgradePacks_NonEmpty(t *testing.T) {
_, ds := runServerWithMockedDS(t)
ds.UserByIDFunc = func(ctx context.Context, id uint) (*fleet.User, error) {
return &fleet.User{ID: id, GlobalRole: ptr.String(fleet.RoleAdmin)}, nil
}
ds.GetPackSpecsFunc = func(ctx context.Context) ([]*fleet.PackSpec, error) {
// queries must match those returned by ListScheduledQueriesInPackWithStats
return []*fleet.PackSpec{
{ID: 1, Name: "p1", Targets: fleet.PackSpecTargets{Labels: []string{"l1"}}, Queries: []fleet.PackSpecQuery{
{QueryName: "q1", Name: "sq1", Interval: 60, Snapshot: ptr.Bool(true), Platform: ptr.String("darwin")},
}},
{ID: 2, Name: "p2", Targets: fleet.PackSpecTargets{Teams: []string{"t1", "t2"}}, Queries: []fleet.PackSpecQuery{
{QueryName: "q2", Name: "sq2", Interval: 90, Removed: ptr.Bool(true), Platform: ptr.String("linux")},
}},
}, nil
}
ds.ListPacksFunc = func(ctx context.Context, opt fleet.PackListOptions) ([]*fleet.Pack, error) {
return []*fleet.Pack{
{ID: 1, Name: "p1", Labels: []fleet.Target{
{Type: fleet.TargetLabel, DisplayText: "l1"},
}, LabelIDs: []uint{1}},
{ID: 2, Name: "p2", Teams: []fleet.Target{
{Type: fleet.TargetTeam, DisplayText: "t1"},
{Type: fleet.TargetTeam, DisplayText: "t2"},
}, TeamIDs: []uint{1, 2}},
}, nil
}
ds.ListScheduledQueriesInPackWithStatsFunc = func(ctx context.Context, id uint, opts fleet.ListOptions) ([]*fleet.ScheduledQuery, error) {
// queries must match those returned by GetPackSpecs
if id == 1 {
return []*fleet.ScheduledQuery{
{ID: 1, PackID: id, Name: "sq1", QueryName: "q1", Interval: 60, Snapshot: ptr.Bool(true), Platform: ptr.String("darwin")},
}, nil
}
return []*fleet.ScheduledQuery{
{ID: 2, PackID: id, Name: "sq2", QueryName: "q2", Interval: 90, Removed: ptr.Bool(true), Platform: ptr.String("linux")},
}, nil
}
ds.CountHostsInTargetsFunc = func(ctx context.Context, filter fleet.TeamFilter, targets fleet.HostTargets, now time.Time) (fleet.TargetMetrics, error) {
return fleet.TargetMetrics{}, nil
}
ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) {
return []*fleet.Query{
{Name: "q1", Query: "select 1"},
{Name: "q2", Query: "select 2"},
{Name: "q3", Query: "select 3"},
}, nil
}
const expected = `
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: (converted from pack "p1", query "q1")
interval: 0
logging: snapshot
min_osquery_version: ""
name: p1 - q1 - Jan 1 00:00:00.000
observer_can_run: false
platform: darwin
query: select 1
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: true
description: (converted from pack "p2", query "q2")
interval: 90
logging: differential
min_osquery_version: ""
name: p2 - q2 - t1 - Jan 1 00:00:00.000
observer_can_run: false
platform: linux
query: select 2
team: t1
---
apiVersion: v1
kind: query
spec:
automations_enabled: true
description: (converted from pack "p2", query "q2")
interval: 90
logging: differential
min_osquery_version: ""
name: p2 - q2 - t2 - Jan 1 00:00:00.000
observer_can_run: false
platform: linux
query: select 2
team: t2
---
`
tempDir := t.TempDir()
outputFile := filepath.Join(tempDir, "output.yml")
// write some dummy data in the file, it should be overwritten
err := os.WriteFile(outputFile, []byte("dummy"), 0644)
require.NoError(t, err)
testUpgradePacksTimestamp = time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
got := runAppForTest(t, []string{"upgrade-packs", "-o", outputFile})
require.Contains(t, got, `Converted 2 queries from 2 2017 "Packs" into portable queries:`)
content, err := os.ReadFile(outputFile)
require.NoError(t, err)
require.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(content)))
}
func TestFleetctlUpgradePacks_NotAdmin(t *testing.T) {
_, ds := runServerWithMockedDS(t)
ds.UserByIDFunc = func(ctx context.Context, id uint) (*fleet.User, error) {
return &fleet.User{ID: id, GlobalRole: ptr.String(fleet.RoleObserver)}, nil
}
tempDir := t.TempDir()
outputFile := filepath.Join(tempDir, "output.yml")
// write some dummy data in the file, it should NOT be overwritten
err := os.WriteFile(outputFile, []byte("dummy"), 0644)
require.NoError(t, err)
// first try without the required output file flag
runAppCheckErr(t, []string{"upgrade-packs"}, `Required flag "o" not set`)
// then try with the required flag but user is not admin
runAppCheckErr(t, []string{"upgrade-packs", "-o", outputFile}, `could not upgrade packs: forbidden: user does not have the admin role`)
content, err := os.ReadFile(outputFile)
require.NoError(t, err)
require.Equal(t, []byte("dummy"), content)
}
func TestFleetctlUpgradePacks_NoPack(t *testing.T) {
_, ds := runServerWithMockedDS(t)
ds.UserByIDFunc = func(ctx context.Context, id uint) (*fleet.User, error) {
return &fleet.User{ID: id, GlobalRole: ptr.String(fleet.RoleAdmin)}, nil
}
ds.GetPackSpecsFunc = func(ctx context.Context) ([]*fleet.PackSpec, error) {
return nil, nil
}
tempDir := t.TempDir()
outputFile := filepath.Join(tempDir, "output.yml")
// write some dummy data in the file, it should NOT be overwritten
err := os.WriteFile(outputFile, []byte("dummy"), 0644)
require.NoError(t, err)
got := runAppForTest(t, []string{"upgrade-packs", "-o", outputFile})
require.Contains(t, got, "No 2017 \"Packs\" found.\n")
content, err := os.ReadFile(outputFile)
require.NoError(t, err)
require.Equal(t, []byte("dummy"), content)
}
+10 -8
View File
@@ -1,6 +1,6 @@
# FAQ
## Using Fleet
## Using Fleet
### How can I switch to Fleet from Kolide Fleet?
@@ -89,7 +89,7 @@ Don't worry, this behavior is expected; it's part of how osquery works.
Fleet and osquery work together by communicating with heartbeats. Depending on how close the next heartbeat is, Fleet might return results a few seconds faster or slower.
>By the way, to get around a phenomena called the "thundering herd problem", these heartbeats aren't exactly the same number of seconds apart each time. osquery implements a "splay", a few ± milliseconds that are added to or subtracted from the heartbeat interval to prevent these thundering herds. This helps prevent situations where many thousands of devices might unnecessarily attempt to communicate with the Fleet server at exactly the same time. (If you've ever used Socket.io, a similar phenomena can occur with that tool's automatic WebSocket reconnects.)
### Why don't my query results appear sorted based upon the ORDER BY clause I specified in my SQL query?
### Why don't my query results appear sorted based upon the ORDER BY clause I specified in my SQL query?
When a query executes in Fleet, the query is sent to all hosts at the same time, but results are returned from hosts at different times. In Fleet, results are shown as soon as Fleet receives a response from a host. Fleet does not sort the overall results across all hosts (the sort UI toggle is used for this). Instead, Fleet prioritizes speed when displaying the results. This means that if you use an `ORDER BY` clause selection criteria in a query, the results may not initially appear with your desired order, however, the sort UI toggle allows you to sort by ascending or descending order for any of the displayed columns.
@@ -335,6 +335,8 @@ There are many challenges to generating .msi packages on any OS but Windows. Err
Packs are a function of osquery that provide a portable format to import /export queries in and out of platforms like Fleet. These osquery packs still exist, but have been removed from the Fleet UI. Access via API is still available for backwards compatibility.
Within Fleet we've introduced the concept of teams in Fleet premium to target specific groups of hosts, but you can also still use scheduled queries in Fleet free (works like packs) to target all your hosts.
The `fleetctl upgrade-packs` command can be used to convert existing packs to queries.
### What happens when I turn off MDM?
In the Fleet UI, you can turn off MDM for a host by selecting **Actions > Turn off MDM** on the **Host details** page.
@@ -451,7 +453,7 @@ Absolutely! If you're updating from the current major release of Fleet (v4), you
If you're updating from an older version (we'll use Fleet v3 as an example), it's best to take some stops along the way:
1. Back up your database.
1. Back up your database.
2. Upgrade to the last release of of v3 - [3.13.0](https://github.com/fleetdm/fleet/releases/tag/3.13.0).
3. Migrate the database.
4. Test
@@ -461,9 +463,9 @@ If you're updating from an older version (we'll use Fleet v3 as an example), it'
8. Test
9. Upgrade to the [current release](https://github.com/fleetdm/fleet/releases/latest).
10. One last migration.
11. Test again for good measure.
11. Test again for good measure.
Taking it a bit slower on major releases gives you an opportunity to better track down where any issues may have been introduced.
Taking it a bit slower on major releases gives you an opportunity to better track down where any issues may have been introduced.
### I upgraded my database, but Fleet is still running slowly. What could be going on?
@@ -531,7 +533,7 @@ If you would like to manage hosts that can travel outside your VPN or intranet w
- `/api/osquery`
- `/api/v1/osquery`
If you are using Fleet Desktop and want it to work on remote devices, the bare minimum API to expose is `/api/latest/fleet/device/*/desktop`. This minimal endpoint will only provide the number of failing policies.
If you are using Fleet Desktop and want it to work on remote devices, the bare minimum API to expose is `/api/latest/fleet/device/*/desktop`. This minimal endpoint will only provide the number of failing policies.
For full Fleet Desktop functionality, `/api/fleet/orbit/*` and`/api/fleet/device/ping` must also be exposed.
@@ -551,7 +553,7 @@ If you would like to use Fleet's MDM features, the following endpoints need to b
download an enrollment profile.
> The `/mdm/apple/scep` and `/mdm/apple/mdm` endpoints are outside of the `/api` path because they
> are not RESTful, and are not intended for use by API clients or browsers.
> are not RESTful, and are not intended for use by API clients or browsers.
### What is the minimum version of MySQL required by Fleet?
@@ -577,4 +579,4 @@ After upgrading to 4.35, any global scheduled query will have its query be conve
<meta name="description" value="Commonly asked questions and answers about deployment from the Fleet community.">
<meta name="description" value="Commonly asked questions and answers about deployment from the Fleet community.">
+20 -4
View File
@@ -15,22 +15,38 @@ func (c *Client) ApplyPacks(specs []*fleet.PackSpec) error {
return c.authenticatedRequest(req, verb, path, &responseBody)
}
// GetPack retrieves information about a pack
func (c *Client) GetPack(name string) (*fleet.PackSpec, error) {
// GetPackSpec retrieves information about a pack in apply spec format.
func (c *Client) GetPackSpec(name string) (*fleet.PackSpec, error) {
verb, path := "GET", "/api/latest/fleet/spec/packs/"+url.PathEscape(name)
var responseBody getPackSpecResponse
err := c.authenticatedRequest(nil, verb, path, &responseBody)
return responseBody.Spec, err
}
// GetPacks retrieves the list of all Packs.
func (c *Client) GetPacks() ([]*fleet.PackSpec, error) {
// GetPacksSpecs retrieves the list of all Packs in apply specs format.
func (c *Client) GetPacksSpecs() ([]*fleet.PackSpec, error) {
verb, path := "GET", "/api/latest/fleet/spec/packs"
var responseBody getPackSpecsResponse
err := c.authenticatedRequest(nil, verb, path, &responseBody)
return responseBody.Specs, err
}
// ListPacks retrieves the list of all Packs.
func (c *Client) ListPacks() ([]*fleet.Pack, error) {
verb, path := "GET", "/api/latest/fleet/packs"
var responseBody listPacksResponse
if err := c.authenticatedRequest(nil, verb, path, &responseBody); err != nil {
return nil, err
}
packs := make([]*fleet.Pack, 0, len(responseBody.Packs))
for _, pr := range responseBody.Packs {
pack := pr.Pack
packs = append(packs, &pack)
}
return packs, nil
}
// DeletePack deletes the pack with the matching name.
func (c *Client) DeletePack(name string) error {
verb, path := "DELETE", "/api/latest/fleet/packs/"+url.PathEscape(name)