Improve vulnerability detection for Ubuntu (#6102)

Feature: Improve our capability to detect vulnerable software on Ubuntu hosts

To improve the capability of detecting vulnerable software on Ubuntu, we are now using OVAL definitions to detect vulnerable software on Ubuntu hosts. If data sync is enabled (disable_data_sync=false) OVAL definitions are automatically kept up to date (they are 'refreshed' once per day) - there's also the option to manually download the OVAL definitions using the 'fleetctl vulnerability-data-stream' command. Downloaded definitions are then parsed into an intermediary format and then used to identify vulnerable software on Ubuntu hosts. Finally, any 'recent' detected vulnerabilities are sent to any third-party integrations.

server/vulnerabilities/oval/input/constant_variable.go

@@ -0,0 +1,8 @@
+package oval_input
+
+// ConstantVariableXML see https://oval.mitre.org/language/version5.10.1/ovaldefinition/documentation/oval-definitions-schema.html#constant_variable.
+type ConstantVariableXML struct {
+	Id       string   `xml:"id,attr"`
+	DataType string   `xml:"datatype,attr"`
+	Values   []string `xml:"value"`
+}

server/vulnerabilities/oval/input/definition.go

@@ -0,0 +1,27 @@
+package oval_input
+
+// CriterionXML see https://oval.mitre.org/language/version5.10.1/ovaldefinition/documentation/oval-definitions-schema.html#CriterionType.
+type CriterionXML struct {
+	TestId string `xml:"test_ref,attr"`
+	Negate string `xml:"negate,attr"`
+}
+
+// CriteriaXML see https://oval.mitre.org/language/version5.10.1/ovaldefinition/documentation/oval-definitions-schema.html#CriteriaType.
+type CriteriaXML struct {
+	Operator   string         `xml:"operator,attr"`
+	Negate     string         `xml:"negate,attr"`
+	Criteriums []CriterionXML `xml:"criterion"`
+	Criterias  []CriteriaXML  `xml:"criteria"`
+}
+
+// ReferenceXML see https://oval.mitre.org/language/version5.10.1/ovaldefinition/documentation/oval-definitions-schema.html#ReferenceType.
+type ReferenceXML struct {
+	Id string `xml:"ref_id,attr"`
+}
+
+// DefinitionXML see https://oval.mitre.org/language/version5.10.1/ovaldefinition/documentation/oval-definitions-schema.html#definition.
+type DefinitionXML struct {
+	Id       string         `xml:"id,attr"`
+	CVEs     []ReferenceXML `xml:"metadata>reference"`
+	Criteria CriteriaXML    `xml:"criteria"`
+}

server/vulnerabilities/oval/input/dpkg_object.go

@@ -0,0 +1,11 @@
+package oval_input
+
+type dpkgObjectNameXML struct {
+	VarRef string `xml:"var_ref,attr"`
+}
+
+// DpkgObjectXML see https://oval.mitre.org/language/version5.10.1/ovaldefinition/documentation/linux-definitions-schema.html#dpkginfo_object.
+type DpkgObjectXML struct {
+	Id   string            `xml:"id,attr"`
+	Name dpkgObjectNameXML `xml:"name"`
+}

server/vulnerabilities/oval/input/dpkg_state.go

@@ -0,0 +1,18 @@
+package oval_input
+
+type ovalSimpleTypeXML struct {
+	Datatype string `xml:"datatype"`
+	Value    string `xml:",chardata"`
+	Op       string `xml:"operation,attr"`
+}
+
+// DpkgStateXML see https://oval.mitre.org/language/version5.10.1/ovaldefinition/documentation/linux-definitions-schema.html#dpkginfo_state.
+type DpkgStateXML struct {
+	Id      string             `xml:"id,attr"`
+	Name    *ovalSimpleTypeXML `xml:"name"`
+	Arch    *ovalSimpleTypeXML `xml:"arch"`
+	Epoch   *ovalSimpleTypeXML `xml:"epoch,omitempty"`
+	Release *ovalSimpleTypeXML `xml:"release,omitempty"`
+	Version *ovalSimpleTypeXML `xml:"version,omitempty"`
+	Evr     *ovalSimpleTypeXML `xml:"evr"`
+}

server/vulnerabilities/oval/input/dpkgtest.go

@@ -0,0 +1,19 @@
+package oval_input
+
+type dpkgTestStateXML struct {
+	Id string `xml:"state_ref,attr"`
+}
+
+type dpkgTestObjectXML struct {
+	Id string `xml:"object_ref,attr"`
+}
+
+// DpkgInfoTestXML see https://oval.mitre.org/language/version5.10.1/ovaldefinition/documentation/linux-definitions-schema.html#dpkginfo_test
+type DpkgInfoTestXML struct {
+	Id             string             `xml:"id,attr"`
+	CheckExistence string             `xml:"check_existence,attr"`
+	Check          string             `xml:"check,attr"`
+	StateOperator  string             `xml:"state_operator,attr"`
+	Object         dpkgTestObjectXML  `xml:"object"`
+	States         []dpkgTestStateXML `xml:"state"`
+}

server/vulnerabilities/oval/input/ubuntu_result.go

@@ -0,0 +1,10 @@
+package oval_input
+
+// UbuntuResultXML groups together the different tokens produced from parsing an OVAL file make for Ubuntu.
+type UbuntuResultXML struct {
+	Definitions    []DefinitionXML
+	PackageTests   []DpkgInfoTestXML
+	PackageStates  []DpkgStateXML
+	PackageObjects []DpkgObjectXML
+	Variables      map[string]ConstantVariableXML
+}