diff --git a/changes/issue-1963-vulnerabilities-no-sync b/changes/issue-1963-vulnerabilities-no-sync new file mode 100644 index 0000000000..c99dd2d4a7 --- /dev/null +++ b/changes/issue-1963-vulnerabilities-no-sync @@ -0,0 +1,2 @@ +* Add fleetctl vulnerability-data-stream command to sync the vulnerabilities processing data streams by hand. +* Add vulnerabilities.disable_data_sync config to fleet serve to avoid downloading the data streams. diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 71e639e92e..4768dc80b5 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -583,13 +583,13 @@ func cronVulnerabilities( } } - err := vulnerabilities.TranslateSoftwareToCPE(ctx, ds, vulnPath, logger, config.Vulnerabilities.CPEDatabaseURL) + err := vulnerabilities.TranslateSoftwareToCPE(ctx, ds, vulnPath, logger, config) if err != nil { level.Error(logger).Log("msg", "analyzing vulnerable software: Software->CPE", "err", err) continue } - err = vulnerabilities.TranslateCPEToCVE(ctx, ds, vulnPath, logger, config.Vulnerabilities.CVEFeedPrefixURL) + err = vulnerabilities.TranslateCPEToCVE(ctx, ds, vulnPath, logger, config) if err != nil { level.Error(logger).Log("msg", "analyzing vulnerable software: CPE->CVE", "err", err) continue diff --git a/cmd/fleetctl/fleetctl.go b/cmd/fleetctl/fleetctl.go index 5738537e95..bcdf943eab 100644 --- a/cmd/fleetctl/fleetctl.go +++ b/cmd/fleetctl/fleetctl.go @@ -46,7 +46,7 @@ func createApp(reader io.Reader, writer io.Writer, exitErrHandler cli.ExitErrHan logoutCommand(), queryCommand(), getCommand(), - &cli.Command{ + { Name: "config", Usage: "Modify Fleet server connection settings", Subcommands: []*cli.Command{ @@ -61,6 +61,7 @@ func createApp(reader io.Reader, writer io.Writer, exitErrHandler cli.ExitErrHan previewCommand(), eefleetctl.UpdatesCommand(), hostsCommand(), + vulnerabilityDataStreamCommand(), packageCommand(), } return app diff --git a/cmd/fleetctl/vulnerability_data_stream.go b/cmd/fleetctl/vulnerability_data_stream.go new file mode 100644 index 0000000000..7cb63fccc9 --- /dev/null +++ b/cmd/fleetctl/vulnerability_data_stream.go @@ -0,0 +1,71 @@ +package main + +import ( + "net/http" + "os" + "path" + + "github.com/fleetdm/fleet/v4/server/config" + "github.com/fleetdm/fleet/v4/server/vulnerabilities" + "github.com/pkg/errors" + "github.com/urfave/cli/v2" +) + +func vulnerabilityDataStreamCommand() *cli.Command { + var dir string + return &cli.Command{ + Name: "vulnerability-data-stream", + Usage: "Download the vulnerability data stream", + UsageText: ` +fleetctl vulnerability-data-stream [options] + +Downloads (if needed) the data streams that can be used by the Fleet server to process software for vulnerabilities. +`, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "dir", + EnvVars: []string{"DIR"}, + Value: "", + Destination: &dir, + Usage: "Directory to place the data streams in", + }, + configFlag(), + contextFlag(), + debugFlag(), + }, + Action: func(c *cli.Context) error { + if dir == "" { + return errors.New("No directory provided") + } + + err := os.MkdirAll(dir, 0o700) + if err != nil { + return err + } + + log(c, "[-] Downloading CPE database...") + + dbPath := path.Join(dir, "cpe.sqlite") + client := &http.Client{} + err = vulnerabilities.SyncCPEDatabase(client, dbPath, config.FleetConfig{}) + if err != nil { + return err + } + + log(c, " Done\n") + + log(c, "[-] Downloading CVE data streams...") + + err = vulnerabilities.SyncCVEData(dir, config.FleetConfig{}) + if err != nil { + return err + } + + log(c, " Done\n") + + log(c, "[+] Data streams successfully downloaded!\n") + + return nil + }, + } +} diff --git a/cmd/fleetctl/vulnerability_data_stream_test.go b/cmd/fleetctl/vulnerability_data_stream_test.go new file mode 100644 index 0000000000..d3c7469eec --- /dev/null +++ b/cmd/fleetctl/vulnerability_data_stream_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "os" + "path" + "path/filepath" + "regexp" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestVulnerabilityDataStream(t *testing.T) { + runAppCheckErr(t, []string{"vulnerability-data-stream"}, "No directory provided") + + vulnPath := t.TempDir() + expected := `[-] Downloading CPE database... Done +[-] Downloading CVE data streams... Done +[+] Data streams successfully downloaded! +` + assert.Equal(t, + expected, + runAppForTest(t, []string{"vulnerability-data-stream", "--dir", vulnPath}), + ) + assert.FileExists(t, path.Join(vulnPath, "cpe.sqlite")) + + var files []string + require.NoError(t, filepath.Walk(vulnPath, func(path string, info os.FileInfo, err error) error { + if match, err := regexp.MatchString("nvdcve.*\\.gz$", path); !match || err != nil { + return nil + } + files = append(files, path) + return nil + })) + assert.Greater(t, len(files), 0) +} diff --git a/docs/1-Using-Fleet/13-Vulnerability-Processing.md b/docs/1-Using-Fleet/13-Vulnerability-Processing.md index 0df6d5950f..70092798c5 100644 --- a/docs/1-Using-Fleet/13-Vulnerability-Processing.md +++ b/docs/1-Using-Fleet/13-Vulnerability-Processing.md @@ -71,4 +71,4 @@ The path specified needs to exist and Fleet needs to be able to read and write t configuration needed for vulnerability processing to work. Additional options, like vulnerability check frequency, can be found in the [configuration documentation](../2-Deploying/2-Configuration.md#vulnerabilities). -You'll need to restart the Fleet instances after changing these settings. +You'll need to restart the Fleet instances after changing these settings. \ No newline at end of file diff --git a/docs/2-Deploying/2-Configuration.md b/docs/2-Deploying/2-Configuration.md index 3a095e96ae..1a41211d35 100644 --- a/docs/2-Deploying/2-Configuration.md +++ b/docs/2-Deploying/2-Configuration.md @@ -1344,6 +1344,22 @@ When running multiple instances of the Fleet server, by default, one of them dyn current_instance_checks: yes ``` +###### disable_data_sync + +Fleet by default automatically downloads and keeps the different data streams needed to properly do vulnerability processing. In some setups, this behavior is not wanted, as access to outside resources might be blocked, or the data stream files might need review/audit before use. + +In order to support vulnerability processing in such environments, we allow users to disable automatic sync of data streams with this configuration value. + +To download the data streams, you can use `fleetctl vulnerability-data-stream --dir ./somedir`. The contents downloaded can then be reviewed, and finally uploaded to the defined `databases_path` in the fleet instance(s) doing the vulnerability processing. + +- Default value: false +- Environment variable: `FLEET_VULNERABILITIES_DISABLE_DATA_SYNC` +- Config file format: + + ``` + vulnerabilities: + disable_data_sync: true + ## Managing osquery configurations We recommend that you use an infrastructure configuration management tool to manage these osquery configurations consistently across your environment. If you're unsure about what configuration management tools your organization uses, contact your company's system administrators. If you are evaluating new solutions for this problem, the founders of Fleet have successfully managed configurations in large production environments using [Chef](https://www.chef.io/chef/) and [Puppet](https://puppet.com/). diff --git a/frontend/components/forms/Form.jsx b/frontend/components/forms/Form.jsx index 5dbd95cecc..d2990b8b05 100644 --- a/frontend/components/forms/Form.jsx +++ b/frontend/components/forms/Form.jsx @@ -133,7 +133,7 @@ export default (WrappedComponent, { fields, validate = defaultValidate }) => { getValue = (fieldName) => { return this.state.formData[fieldName]; }; - + resetField = (fieldName) => { const { errors, formData } = this.state; diff --git a/server/config/config.go b/server/config/config.go index 7c770a266c..e687581014 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -171,6 +171,7 @@ type VulnerabilitiesConfig struct { CPEDatabaseURL string `json:"cpe_database_url" yaml:"cpe_database_url"` CVEFeedPrefixURL string `json:"cve_feed_prefix_url" yaml:"cve_feed_prefix_url"` CurrentInstanceChecks string `json:"current_instance_checks" yaml:"current_instance_checks"` + DisableDataSync bool `json:"disable_data_sync" yaml:"disable_data_sync"` } // FleetConfig stores the application configuration. Each subcategory is @@ -384,6 +385,8 @@ func (man Manager) addConfigs() { "Prefix URL for the CVE data feed. If empty, default to https://nvd.nist.gov/") man.addConfigString("vulnerabilities.current_instance_checks", "auto", "Allows to manually select an instance to do the vulnerability processing.") + man.addConfigBool("vulnerabilities.disable_data_sync", false, + "Skips synchronizing data streams and expects them to be available in the databases_path.") } // LoadConfig will load the config variables into a fully initialized @@ -514,6 +517,7 @@ func (man Manager) LoadConfig() FleetConfig { CPEDatabaseURL: man.getConfigString("vulnerabilities.cpe_database_url"), CVEFeedPrefixURL: man.getConfigString("vulnerabilities.cve_feed_prefix_url"), CurrentInstanceChecks: man.getConfigString("vulnerabilities.current_instance_checks"), + DisableDataSync: man.getConfigBool("vulnerabilities.disable_data_sync"), }, } } diff --git a/server/vulnerabilities/cpe.go b/server/vulnerabilities/cpe.go index 3ffcdbd50d..4f0e734d12 100644 --- a/server/vulnerabilities/cpe.go +++ b/server/vulnerabilities/cpe.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/fleet" kitlog "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -66,8 +67,16 @@ func GetLatestNVDRelease(client *http.Client) (*NVDRelease, error) { }, nil } -func syncCPEDatabase(client *http.Client, dbPath string, cpeDatabaseURLOverride string) error { - url := cpeDatabaseURLOverride +func SyncCPEDatabase( + client *http.Client, + dbPath string, + config config.FleetConfig, +) error { + if config.Vulnerabilities.DisableDataSync { + return nil + } + + url := config.Vulnerabilities.CPEDatabaseURL if url == "" { nvdRelease, err := GetLatestNVDRelease(client) if err != nil { @@ -212,12 +221,12 @@ func TranslateSoftwareToCPE( ds fleet.Datastore, vulnPath string, logger kitlog.Logger, - cpeDatabaseURLOverride string, + config config.FleetConfig, ) error { dbPath := path.Join(vulnPath, "cpe.sqlite") client := &http.Client{} - if err := syncCPEDatabase(client, dbPath, cpeDatabaseURLOverride); err != nil { + if err := SyncCPEDatabase(client, dbPath, config); err != nil { return errors.Wrap(err, "sync cpe db") } diff --git a/server/vulnerabilities/cpe_test.go b/server/vulnerabilities/cpe_test.go index b74c6e2d61..e0e35e9170 100644 --- a/server/vulnerabilities/cpe_test.go +++ b/server/vulnerabilities/cpe_test.go @@ -15,6 +15,7 @@ import ( "github.com/dnaeon/go-vcr/v2/recorder" "github.com/facebookincubator/nvdtools/cpedict" + "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/mock" kitlog "github.com/go-kit/kit/log" @@ -70,7 +71,7 @@ func TestSyncCPEDatabase(t *testing.T) { } // first time, db doesn't exist, so it downloads - err = syncCPEDatabase(client, dbPath, "") + err = SyncCPEDatabase(client, dbPath, config.FleetConfig{}) require.NoError(t, err) db, err := sqliteDB(dbPath) @@ -94,7 +95,7 @@ func TestSyncCPEDatabase(t *testing.T) { require.NoError(t, err) // then it will download - err = syncCPEDatabase(client, dbPath, "") + err = SyncCPEDatabase(client, dbPath, config.FleetConfig{}) require.NoError(t, err) // let's register the mtime for the db @@ -115,7 +116,7 @@ func TestSyncCPEDatabase(t *testing.T) { time.Sleep(2 * time.Second) // let's check it doesn't download because it's new enough - err = syncCPEDatabase(client, dbPath, "") + err = SyncCPEDatabase(client, dbPath, config.FleetConfig{}) require.NoError(t, err) stat, err = os.Stat(dbPath) @@ -184,7 +185,7 @@ func TestTranslateSoftwareToCPE(t *testing.T) { err = GenerateCPEDB(dbPath, items) require.NoError(t, err) - err = TranslateSoftwareToCPE(context.Background(), ds, tempDir, kitlog.NewNopLogger(), "") + err = TranslateSoftwareToCPE(context.Background(), ds, tempDir, kitlog.NewNopLogger(), config.FleetConfig{}) require.NoError(t, err) assert.Equal(t, []string{ "cpe:2.3:a:vendor:product-1:1.2.3:*:*:*:*:macos:*:*", @@ -208,10 +209,28 @@ func TestSyncsCPEFromURL(t *testing.T) { tempDir := t.TempDir() dbPath := path.Join(tempDir, "cpe.sqlite") - err := syncCPEDatabase(client, dbPath, ts.URL) + err := SyncCPEDatabase( + client, dbPath, config.FleetConfig{Vulnerabilities: config.VulnerabilitiesConfig{CPEDatabaseURL: ts.URL}}) require.NoError(t, err) stored, err := ioutil.ReadFile(dbPath) require.NoError(t, err) assert.Equal(t, "Hello world!", string(stored)) } + +func TestSyncsCPESkipsIfDisableSync(t *testing.T) { + client := &http.Client{} + tempDir := t.TempDir() + dbPath := path.Join(tempDir, "cpe.sqlite") + + fleetConfig := config.FleetConfig{ + Vulnerabilities: config.VulnerabilitiesConfig{ + DisableDataSync: true, + }, + } + err := SyncCPEDatabase(client, dbPath, fleetConfig) + require.NoError(t, err) + + _, err = os.Stat(dbPath) + require.ErrorIs(t, err, os.ErrNotExist) +} diff --git a/server/vulnerabilities/cve.go b/server/vulnerabilities/cve.go index d0f1b93f75..2ff9437386 100644 --- a/server/vulnerabilities/cve.go +++ b/server/vulnerabilities/cve.go @@ -14,18 +14,23 @@ import ( "github.com/facebookincubator/nvdtools/cvefeed" "github.com/facebookincubator/nvdtools/providers/nvd" "github.com/facebookincubator/nvdtools/wfn" + "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/fleet" kitlog "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" ) -func syncCVEData(vulnPath string, cveFeedURLPrefixOverride string) error { +func SyncCVEData(vulnPath string, config config.FleetConfig) error { + if config.Vulnerabilities.DisableDataSync { + return nil + } + cve := nvd.SupportedCVE["cve-1.1.json.gz"] source := nvd.NewSourceConfig() - if cveFeedURLPrefixOverride != "" { - parsed, err := url.Parse(cveFeedURLPrefixOverride) + if config.Vulnerabilities.CVEFeedPrefixURL != "" { + parsed, err := url.Parse(config.Vulnerabilities.CVEFeedPrefixURL) if err != nil { return errors.Wrap(err, "parsing cve feed url prefix override") } @@ -50,9 +55,9 @@ func TranslateCPEToCVE( ds fleet.Datastore, vulnPath string, logger kitlog.Logger, - cveFeedURLPrefixOverride string, + config config.FleetConfig, ) error { - err := syncCVEData(vulnPath, cveFeedURLPrefixOverride) + err := SyncCVEData(vulnPath, config) if err != nil { return err } diff --git a/server/vulnerabilities/cve_test.go b/server/vulnerabilities/cve_test.go index 58f9a26530..adc83e8082 100644 --- a/server/vulnerabilities/cve_test.go +++ b/server/vulnerabilities/cve_test.go @@ -6,10 +6,13 @@ import ( "net/http" "net/http/httptest" "os" + "path/filepath" + "regexp" "strings" "sync" "testing" + "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/mock" kitlog "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" @@ -45,7 +48,7 @@ func TestTranslateCPEToCVE(t *testing.T) { return nil } - err := TranslateCPEToCVE(ctx, ds, tempDir, kitlog.NewLogfmtLogger(os.Stdout), "") + err := TranslateCPEToCVE(ctx, ds, tempDir, kitlog.NewLogfmtLogger(os.Stdout), config.FleetConfig{}) require.NoError(t, err) require.Equal(t, []string{tt.cve}, cvesFound) @@ -55,7 +58,6 @@ func TestTranslateCPEToCVE(t *testing.T) { } func TestSyncsCVEFromURL(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.RequestURI, ".meta") { fmt.Fprint(w, "lastModifiedDate:2021-08-04T11:10:30-04:00\r\n") @@ -68,10 +70,32 @@ func TestSyncsCVEFromURL(t *testing.T) { defer ts.Close() tempDir := t.TempDir() - err := syncCVEData(tempDir, ts.URL) + err := SyncCVEData( + tempDir, config.FleetConfig{Vulnerabilities: config.VulnerabilitiesConfig{CVEFeedPrefixURL: ts.URL}}) require.Error(t, err) require.Equal(t, fmt.Sprintf("1 synchronisation error:\n\tunexpected size for \"%s/feeds/json/cve/1.1/nvdcve-1.1-2002.json.gz\" (200 OK): want 1453293, have 0", ts.URL), err.Error(), ) } + +func TestSyncsCVEFromURLSkipsIfDisableSync(t *testing.T) { + tempDir := t.TempDir() + fleetConfig := config.FleetConfig{ + Vulnerabilities: config.VulnerabilitiesConfig{ + DisableDataSync: true, + }, + } + err := SyncCVEData(tempDir, fleetConfig) + require.NoError(t, err) + err = filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error { + if match, err := regexp.MatchString("nvdcve.*\\.gz$", path); !match || err != nil { + return nil + } + + t.FailNow() + + return nil + }) + require.NoError(t, err) +}