Discharge7443/fleet
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

Fleet server and tooling to use NETWORK_TEST_GITHUB_TOKEN when environment variable is set. (#9143)

Browse Source 
* WIP

* Add more logging

* Check rate limit at end of action

* Add github client in more places

* Add new published firefox 93 vulnerabilities to tests

* Remove fmt printfs

* Restore CI check settings

* Readd newline
This commit is contained in:
Lucas Manuel Rodriguez
2023-01-03 14:56:11 -03:00
committed by GitHub
parent b990121fdc
commit ac22aadc13
14 changed files with 172 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/vulnerabilities/msrc/sync.go
+6 -4
View File
@@ -3,8 +3,8 @@ package msrc
import (
"context"
"fmt"
"net/http"
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/io"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/parsed"
@@ -58,10 +58,12 @@ func bulletinsDelta(
return toDownload, toDelete
}
// Sync syncs the local msrc security bulletins (contained in dstDir) for one or more operating systems with the security
// bulletin published in Github.
// SyncFromGithub syncs the local msrc security bulletins (contained in dstDir) for one or more operating
// systems with the security bulletin published in Github.
//
// If 'os' is nil, then all security bulletins will be synched.
func Sync(ctx context.Context, client *http.Client, dstDir string, os []fleet.OperatingSystem) error {
func SyncFromGithub(ctx context.Context, dstDir string, os []fleet.OperatingSystem) error {
client := fleethttp.NewGithubClient()
rep := github.NewClient(client).Repositories
gh := io.NewGitHubClient(client, rep, dstDir)
fs := io.NewFSClient(dstDir)
server/vulnerabilities/nvd/cpe.go
+11 -12
View File
@@ -5,7 +5,6 @@ import (
"database/sql"
"errors"
"fmt"
"net/http"
"net/url"
"os"
"path/filepath"
@@ -14,6 +13,7 @@ import (
"github.com/doug-martin/goqu/v9"
"github.com/fleetdm/fleet/v4/pkg/download"
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/oval"
@@ -31,10 +31,11 @@ const (
var cpeDBRegex = regexp.MustCompile(`^cpe-.*\.sqlite\.gz$`)
func GetLatestNVDRelease(client *http.Client) (*github.RepositoryRelease, error) {
ghclient := github.NewClient(client)
ctx := context.Background()
releases, _, err := ghclient.Repositories.ListReleases(ctx, owner, repo, &github.ListOptions{Page: 0, PerPage: 10})
func GetLatestGithubNVDRelease() (*github.RepositoryRelease, error) {
githubClient := github.NewClient(fleethttp.NewGithubClient())
releases, _, err := githubClient.Repositories.ListReleases(
context.Background(), owner, repo, &github.ListOptions{Page: 0, PerPage: 10},
)
if err != nil {
return nil, err
}
@@ -51,15 +52,11 @@ func GetLatestNVDRelease(client *http.Client) (*github.RepositoryRelease, error)
// DownloadCPEDB downloads the CPE database to the given vulnPath. If cpeDBURL is empty, attempts to download it
// from the latest release of github.com/fleetdm/nvd. Skips downloading if CPE database is newer than the release.
func DownloadCPEDB(
vulnPath string,
client *http.Client,
cpeDBURL string,
) error {
func DownloadCPEDBFromGithub(vulnPath string, cpeDBURL string) error {
path := filepath.Join(vulnPath, cpeDBFilename)
if cpeDBURL == "" {
release, err := GetLatestNVDRelease(client)
release, err := GetLatestGithubNVDRelease()
if err != nil {
return err
}
@@ -91,7 +88,9 @@ func DownloadCPEDB(
if err != nil {
return err
}
if err := download.DownloadAndExtract(client, u, path); err != nil {
githubClient := fleethttp.NewGithubClient()
if err := download.DownloadAndExtract(githubClient, u, path); err != nil {
return err
}
server/vulnerabilities/nvd/cpe_test.go
+6 -12
View File
@@ -13,7 +13,6 @@ import (
"time"
"github.com/facebookincubator/nvdtools/cpedict"
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
"github.com/fleetdm/fleet/v4/pkg/nettest"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/mock"
@@ -148,12 +147,10 @@ func TestCPETranslations(t *testing.T) {
func TestSyncCPEDatabase(t *testing.T) {
nettest.Run(t)
client := fleethttp.NewClient()
tempDir := t.TempDir()
// first time, db doesn't exist, so it downloads
err := DownloadCPEDB(tempDir, client, "")
err := DownloadCPEDBFromGithub(tempDir, "")
require.NoError(t, err)
dbPath := filepath.Join(tempDir, "cpe.sqlite")
@@ -193,7 +190,7 @@ func TestSyncCPEDatabase(t *testing.T) {
require.NoError(t, err)
// then it will download
err = DownloadCPEDB(tempDir, client, "")
err = DownloadCPEDBFromGithub(tempDir, "")
require.NoError(t, err)
// let's register the mtime for the db
@@ -214,7 +211,7 @@ func TestSyncCPEDatabase(t *testing.T) {
time.Sleep(2 * time.Second)
// let's check it doesn't download because it's new enough
err = DownloadCPEDB(tempDir, client, "")
err = DownloadCPEDBFromGithub(tempDir, "")
require.NoError(t, err)
stat, err = os.Stat(dbPath)
require.NoError(t, err)
@@ -303,9 +300,8 @@ func TestSyncsCPEFromURL(t *testing.T) {
}))
defer ts.Close()
client := fleethttp.NewClient()
tempDir := t.TempDir()
err := DownloadCPEDB(tempDir, client, ts.URL+"/hello-world.gz")
err := DownloadCPEDBFromGithub(tempDir, ts.URL+"/hello-world.gz")
require.NoError(t, err)
dbPath := filepath.Join(tempDir, "cpe.sqlite")
@@ -1139,18 +1135,16 @@ func TestCPEFromSoftwareIntegration(t *testing.T) {
}
nettest.Run(t)
client := fleethttp.NewClient()
tempDir := t.TempDir()
err := DownloadCPEDB(tempDir, client, "")
err := DownloadCPEDBFromGithub(tempDir, "")
require.NoError(t, err)
dbPath := filepath.Join(tempDir, "cpe.sqlite")
db, err := sqliteDB(dbPath)
require.NoError(t, err)
err = DownloadCPETranslations(tempDir, client, "")
err = DownloadCPETranslationsFromGithub(tempDir, "")
require.NoError(t, err)
cpeTranslationsPath := filepath.Join(".", cpeTranslationsFilename)
server/vulnerabilities/nvd/cpe_translations.go
+5 -4
View File
@@ -4,13 +4,13 @@ import (
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
"github.com/fleetdm/fleet/v4/pkg/download"
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
"github.com/fleetdm/fleet/v4/server/fleet"
)
@@ -31,13 +31,13 @@ func loadCPETranslations(path string) (CPETranslations, error) {
return translations, nil
}
// DownloadCPETranslations downloads the CPE translations to the given vulnPath. If cpeTranslationsURL is empty, attempts to download it
// DownloadCPETranslationsFromGithub downloads the CPE translations to the given vulnPath. If cpeTranslationsURL is empty, attempts to download it
// from the latest release of github.com/fleetdm/nvd. Skips downloading if CPE translations is newer than the release.
func DownloadCPETranslations(vulnPath string, client *http.Client, cpeTranslationsURL string) error {
func DownloadCPETranslationsFromGithub(vulnPath string, cpeTranslationsURL string) error {
path := filepath.Join(vulnPath, cpeTranslationsFilename)
if cpeTranslationsURL == "" {
release, err := GetLatestNVDRelease(client)
release, err := GetLatestGithubNVDRelease()
if err != nil {
return err
}
@@ -69,6 +69,7 @@ func DownloadCPETranslations(vulnPath string, client *http.Client, cpeTranslatio
if err != nil {
return err
}
client := fleethttp.NewGithubClient()
if err := download.Download(client, u, path); err != nil {
return err
}
server/vulnerabilities/nvd/cve_test.go
+91 -48
View File
@@ -19,6 +19,94 @@ import (
"github.com/stretchr/testify/require"
)
// firefox93WindowsVulnerabilities was manually generated by visiting:
// https://nvd.nist.gov/vuln/search/results?form_type=Advanced&results_type=overview&isCpeNameSearch=true&seach_type=all&query=cpe:2.3:a:mozilla:firefox:93.0:*:*:*:*:*:*:*
var firefox93WindowsVulnerabilities = []string{
"CVE-2021-43540",
"CVE-2021-38503",
"CVE-2021-38504",
"CVE-2021-38506",
"CVE-2021-38507",
"CVE-2021-38508",
"CVE-2021-38509",
"CVE-2021-43534",
"CVE-2021-43532",
"CVE-2021-43531",
"CVE-2021-43533",
"CVE-2021-43538",
"CVE-2021-43542",
"CVE-2021-43543",
"CVE-2021-30547",
"CVE-2021-43546",
"CVE-2021-43537",
"CVE-2021-43541",
"CVE-2021-43536",
"CVE-2021-43545",
"CVE-2021-43539",
// These were published on Dec 22nd, 2022.
"CVE-2022-34480",
"CVE-2022-26387",
"CVE-2022-22759",
"CVE-2022-28281",
"CVE-2022-45415",
"CVE-2022-42930",
"CVE-2022-0511",
"CVE-2022-22763",
"CVE-2022-22737",
"CVE-2022-22751",
"CVE-2022-38478",
"CVE-2022-22761",
"CVE-2022-34482",
"CVE-2022-26486",
"CVE-2022-22739",
"CVE-2022-22755",
"CVE-2022-22757",
"CVE-2022-1097",
"CVE-2022-22754",
"CVE-2022-22748",
"CVE-2022-22736",
"CVE-2022-22745",
"CVE-2022-26385",
"CVE-2022-26383",
"CVE-2022-3266",
"CVE-2022-34468",
"CVE-2022-34481",
"CVE-2022-28289",
"CVE-2022-22741",
"CVE-2022-28284",
"CVE-2022-34484",
"CVE-2022-22752",
"CVE-2022-26485",
"CVE-2022-28286",
"CVE-2022-28283",
"CVE-2022-28285",
"CVE-2022-0843",
"CVE-2022-29909",
"CVE-2022-22749",
"CVE-2022-26384",
"CVE-2022-28282",
"CVE-2022-28287",
"CVE-2022-40956",
"CVE-2022-22740",
"CVE-2022-22743",
"CVE-2022-22764",
"CVE-2022-22738",
"CVE-2022-1529",
"CVE-2022-22760",
"CVE-2022-29916",
"CVE-2022-29917",
"CVE-2022-22747",
"CVE-2022-26382",
"CVE-2022-22742",
"CVE-2022-28288",
"CVE-2022-22756",
"CVE-2022-26381",
"CVE-2022-1802",
"CVE-2022-34483",
}
var cvetests = []struct {
cpe string
cves []string
@@ -40,57 +128,11 @@ var cvetests = []struct {
},
{
"cpe:2.3:a:mozilla:firefox:93.0:*:*:*:*:windows:*:*",
[]string{
"CVE-2021-43540",
"CVE-2021-38503",
"CVE-2021-38504",
"CVE-2021-38506",
"CVE-2021-38507",
"CVE-2021-38508",
"CVE-2021-38509",
"CVE-2021-43534",
"CVE-2021-43532",
"CVE-2021-43531",
"CVE-2021-43533",
"CVE-2021-43538",
"CVE-2021-43542",
"CVE-2021-43543",
"CVE-2021-30547",
"CVE-2021-43546",
"CVE-2021-43537",
"CVE-2021-43541",
"CVE-2021-43536",
"CVE-2021-43545",
"CVE-2021-43539",
},
firefox93WindowsVulnerabilities,
},
{
"cpe:2.3:a:mozilla:firefox:93.0.100:*:*:*:*:windows:*:*",
[]string{
"CVE-2021-43540",
"CVE-2021-38503",
"CVE-2021-38504",
"CVE-2021-38506",
"CVE-2021-38507",
"CVE-2021-38508",
"CVE-2021-38509",
"CVE-2021-43534",
"CVE-2021-43532",
"CVE-2021-43531",
"CVE-2021-43533",
"CVE-2021-43538",
"CVE-2021-43542",
"CVE-2021-43543",
"CVE-2021-30547",
"CVE-2021-43546",
"CVE-2021-43537",
"CVE-2021-43541",
"CVE-2021-43536",
"CVE-2021-43545",
"CVE-2021-43539",
},
firefox93WindowsVulnerabilities,
},
}
@@ -152,6 +194,7 @@ func TestTranslateCPEToCVE(t *testing.T) {
ds.InsertSoftwareVulnerabilitiesFunc = func(ctx context.Context, vulns []fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (int64, error) {
cveLock.Lock()
defer cveLock.Unlock()
for _, v := range vulns {
cvesFound = append(cvesFound, v.CVE)
}
server/vulnerabilities/nvd/sync.go
+8 -9
View File
@@ -6,7 +6,6 @@ import (
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
@@ -33,13 +32,11 @@ type SyncOptions struct {
// Sync downloads all the vulnerability data sources.
func Sync(opts SyncOptions) error {
client := fleethttp.NewClient()
if err := DownloadCPEDB(opts.VulnPath, client, opts.CPEDBURL); err != nil {
if err := DownloadCPEDBFromGithub(opts.VulnPath, opts.CPEDBURL); err != nil {
return fmt.Errorf("sync CPE database: %w", err)
}
if err := DownloadCPETranslations(opts.VulnPath, client, opts.CPETranslationsURL); err != nil {
if err := DownloadCPETranslationsFromGithub(opts.VulnPath, opts.CPETranslationsURL); err != nil {
return fmt.Errorf("sync CPE translations: %w", err)
}
@@ -47,11 +44,11 @@ func Sync(opts SyncOptions) error {
return fmt.Errorf("sync NVD CVE feed: %w", err)
}
if err := DownloadEPSSFeed(opts.VulnPath, client); err != nil {
if err := DownloadEPSSFeed(opts.VulnPath); err != nil {
return fmt.Errorf("sync EPSS CVE feed: %w", err)
}
if err := DownloadCISAKnownExploitsFeed(opts.VulnPath, client); err != nil {
if err := DownloadCISAKnownExploitsFeed(opts.VulnPath); err != nil {
return fmt.Errorf("sync CISA known exploits feed: %w", err)
}
@@ -64,7 +61,7 @@ const (
)
// DownloadEPSSFeed downloads the EPSS scores feed.
func DownloadEPSSFeed(vulnPath string, client *http.Client) error {
func DownloadEPSSFeed(vulnPath string) error {
urlString := epssFeedsURL + "/" + epssFilename
u, err := url.Parse(urlString)
if err != nil {
@@ -72,6 +69,7 @@ func DownloadEPSSFeed(vulnPath string, client *http.Client) error {
}
path := filepath.Join(vulnPath, strings.TrimSuffix(epssFilename, ".gz"))
client := fleethttp.NewClient()
err = download.DownloadAndExtract(client, u, path)
if err != nil {
return fmt.Errorf("download %s: %w", u, err)
@@ -160,7 +158,7 @@ type knownExploitedVulnerability struct {
}
// DownloadCISAKnownExploitsFeed downloads the CISA known exploited vulnerabilities feed.
func DownloadCISAKnownExploitsFeed(vulnPath string, client *http.Client) error {
func DownloadCISAKnownExploitsFeed(vulnPath string) error {
path := filepath.Join(vulnPath, cisaKnownExploitsFilename)
u, err := url.Parse(cisaKnownExploitsURL)
@@ -168,6 +166,7 @@ func DownloadCISAKnownExploitsFeed(vulnPath string, client *http.Client) error {
return err
}
client := fleethttp.NewClient()
err = download.Download(client, u, path)
if err != nil {
return fmt.Errorf("download cisa known exploits: %w", err)
server/vulnerabilities/nvd/sync_test.go
+3 -10
View File
@@ -6,7 +6,6 @@ import (
"strings"
"testing"
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
"github.com/fleetdm/fleet/v4/pkg/nettest"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/mock"
@@ -18,11 +17,9 @@ import (
func TestDownloadEPSSFeed(t *testing.T) {
nettest.Run(t)
client := fleethttp.NewClient()
tempDir := t.TempDir()
err := DownloadEPSSFeed(tempDir, client)
err := DownloadEPSSFeed(tempDir)
require.NoError(t, err)
assert.FileExists(t, filepath.Join(tempDir, strings.TrimSuffix(epssFilename, ".gz")))
@@ -31,11 +28,9 @@ func TestDownloadEPSSFeed(t *testing.T) {
func TestDownloadCISAKnownExploitsFeed(t *testing.T) {
nettest.Run(t)
client := fleethttp.NewClient()
tempDir := t.TempDir()
err := DownloadCISAKnownExploitsFeed(tempDir, client)
err := DownloadCISAKnownExploitsFeed(tempDir)
require.NoError(t, err)
assert.FileExists(t, filepath.Join(tempDir, cisaKnownExploitsFilename))
@@ -75,11 +70,9 @@ func TestLoadCVEMeta(t *testing.T) {
func TestDownloadCPETranslations(t *testing.T) {
nettest.Run(t)
client := fleethttp.NewClient()
tempDir := t.TempDir()
err := DownloadCPETranslations(tempDir, client, "")
err := DownloadCPETranslationsFromGithub(tempDir, "")
require.NoError(t, err)
assert.FileExists(t, filepath.Join(tempDir, cpeTranslationsFilename))
server/vulnerabilities/oval/sync.go
+8 -6
View File
@@ -12,14 +12,16 @@ import (
"time"
"github.com/fleetdm/fleet/v4/pkg/download"
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/google/go-github/v37/github"
)
func ghNvdFileGetter(client *http.Client) func(string) (io.ReadCloser, error) {
func ghNvdFileGetter() func(string) (io.ReadCloser, error) {
ghClient := fleethttp.NewGithubClient()
return func(file string) (io.ReadCloser, error) {
src, r, err := github.NewClient(client).Repositories.DownloadContents(
src, r, err := github.NewClient(ghClient).Repositories.DownloadContents(
context.Background(), "fleetdm", "nvd", file, nil)
if err != nil {
return nil, err
@@ -84,8 +86,8 @@ func removeOldDefs(date time.Time, path string) (map[string]bool, error) {
// Sync syncs the oval definitions for one or more platforms.
// If 'platforms' is nil, then all supported platforms will be synched.
func Sync(client *http.Client, dstDir string, platforms []Platform) error {
sources, err := getOvalSources(ghNvdFileGetter(client))
func Sync(dstDir string, platforms []Platform) error {
sources, err := getOvalSources(ghNvdFileGetter())
if err != nil {
return err
}
@@ -96,6 +98,7 @@ func Sync(client *http.Client, dstDir string, platforms []Platform) error {
}
}
client := fleethttp.NewClient()
dwn := downloadDecompressed(client)
for _, platform := range platforms {
defFile, err := downloadDefinitions(sources, platform, dwn)
@@ -123,7 +126,6 @@ func Sync(client *http.Client, dstDir string, platforms []Platform) error {
// Returns a slice of Platforms of the newly downloaded OVAL files.
func Refresh(
ctx context.Context,
client *http.Client,
versions *fleet.OSVersions,
vulnPath string,
) ([]Platform, error) {
@@ -136,7 +138,7 @@ func Refresh(
toDownload := whatToDownload(versions, existing, now)
if len(toDownload) > 0 {
err = Sync(client, vulnPath, toDownload)
err = Sync(vulnPath, toDownload)
if err != nil {
return nil, err
}