- Agent options configure the osquery agent. When you update agent
- options, they will be applied the next time a host checks in to Fleet.
+ Agent options configure Fleet's agent (fleetd). When you update
+ agent options, they will be applied the next time a host checks in to
+ Fleet.
Add hosts
diff --git a/frontend/pages/labels/EditLabelPage/EditLabelPage.tests.tsx b/frontend/pages/labels/EditLabelPage/EditLabelPage.tests.tsx
new file mode 100644
index 0000000000..15a4e53e08
--- /dev/null
+++ b/frontend/pages/labels/EditLabelPage/EditLabelPage.tests.tsx
@@ -0,0 +1,75 @@
+import React from "react";
+
+import { screen } from "@testing-library/react";
+import { createCustomRenderer } from "test/test-utils";
+import mockServer from "test/mock-server";
+import { getLabelHandler } from "test/handlers/label-handlers";
+
+import EditLabelPage from "./EditLabelPage";
+
+// TODO: make this a utility for other tests.
+const generateMockRouterProps = (overrides?: any) => {
+ return {
+ location: {},
+ params: {},
+ route: {},
+ router: [],
+ routeParams: {},
+ ...overrides,
+ };
+};
+
+describe("EditLabelPage", () => {
+ it("renders a message for build in labels", async () => {
+ mockServer.use(getLabelHandler({ label_type: "builtin" }));
+ const render = createCustomRenderer({ withBackendMock: true });
+
+ const routerProps = generateMockRouterProps({
+ routeParams: { label_id: "1" },
+ });
+ render();
+
+ // waiting for the message to render
+ const builtinMessage = await screen.findByText(
+ "Built in labels cannot be edited"
+ );
+
+ expect(builtinMessage).toBeInTheDocument();
+ });
+
+ it("renders the DynamicLabelForm when the label is dynamic", async () => {
+ mockServer.use(getLabelHandler({ label_membership_type: "dynamic" }));
+ const render = createCustomRenderer({ withBackendMock: true });
+
+ const routerProps = generateMockRouterProps({
+ routeParams: { label_id: "1" },
+ });
+ render();
+
+ // waiting for the message to render
+ const queryLabel = await screen.findByText("Query");
+ const platformLabel = await screen.findByText("Platform");
+
+ expect(queryLabel).toBeInTheDocument();
+ expect(platformLabel).toBeInTheDocument();
+ expect(screen.getByText(/Label queries are immutable/)).toBeInTheDocument();
+ expect(
+ screen.getByText(/Label platforms are immutable/)
+ ).toBeInTheDocument();
+ });
+
+ it("renders the ManualLabelForm when the label is manual", async () => {
+ mockServer.use(getLabelHandler({ label_membership_type: "manual" }));
+ const render = createCustomRenderer({ withBackendMock: true });
+
+ const routerProps = generateMockRouterProps({
+ routeParams: { label_id: "1" },
+ });
+ render();
+
+ // waiting for the message to render
+ const selectHostsLabel = await screen.findByText("Select hosts");
+
+ expect(selectHostsLabel).toBeInTheDocument();
+ });
+});
diff --git a/frontend/pages/labels/components/DynamicLabelForm/DynamicLabelForm.tests.tsx b/frontend/pages/labels/components/DynamicLabelForm/DynamicLabelForm.tests.tsx
new file mode 100644
index 0000000000..13cee290fa
--- /dev/null
+++ b/frontend/pages/labels/components/DynamicLabelForm/DynamicLabelForm.tests.tsx
@@ -0,0 +1,45 @@
+import React from "react";
+import { noop } from "lodash";
+import { render, screen } from "@testing-library/react";
+
+import { renderWithSetup } from "test/test-utils";
+
+import DynamicLabelForm from "./DynamicLabelForm";
+
+describe("DynamicLabelForm", () => {
+ it("should render the Fleet Ace and Select Platform input", () => {
+ render();
+
+ expect(screen.getByText("Query")).toBeInTheDocument();
+ expect(screen.getByText("All platforms")).toBeInTheDocument();
+ });
+
+ it("should pass up the form data when the form is submitted and valid", async () => {
+ const onSave = jest.fn();
+
+ const name = "Test Name";
+ const description = "Test Description";
+ const query = "SELECT * FROM users;";
+ const platform = "darwin";
+
+ const { user } = renderWithSetup(
+
+ );
+
+ await user.type(screen.getByLabelText("Name"), name);
+ await user.type(screen.getByLabelText("Description"), description);
+ await user.click(screen.getByRole("button", { name: "Save" }));
+
+ expect(onSave).toHaveBeenCalledWith({
+ name,
+ description,
+ query,
+ platform,
+ });
+ });
+});
diff --git a/frontend/pages/labels/components/LabelForm/LabelForm.tests.tsx b/frontend/pages/labels/components/LabelForm/LabelForm.tests.tsx
new file mode 100644
index 0000000000..719e8ee69c
--- /dev/null
+++ b/frontend/pages/labels/components/LabelForm/LabelForm.tests.tsx
@@ -0,0 +1,57 @@
+import React from "react";
+import { renderWithSetup } from "test/test-utils";
+import { screen, render } from "@testing-library/react";
+import { noop } from "lodash";
+
+// @ts-ignore
+import InputField from "components/forms/fields/InputField";
+
+import LabelForm from "./LabelForm";
+
+describe("LabelForm", () => {
+ it("should validate the name to be required", async () => {
+ const { user } = renderWithSetup(
+
+ );
+
+ const nameInput = screen.getByLabelText("Name");
+
+ await user.click(screen.getByRole("button", { name: "Save" }));
+ expect(screen.getByText("Label name must be present")).toBeInTheDocument();
+
+ await user.type(nameInput, "Label name");
+ expect(
+ screen.queryByText("Label name must be present")
+ ).not.toBeInTheDocument();
+ });
+
+ it("should render any additional field the user provides", () => {
+ render(
+ }
+ />
+ );
+
+ expect(screen.getByLabelText("test field")).toBeInTheDocument();
+ });
+
+ it("should pass up the form data when the form is submitted and valid", async () => {
+ const onSave = jest.fn();
+ const { user } = renderWithSetup(
+
+ );
+
+ const nameValue = "Test Name";
+ const descriptionValue = "Test Description";
+ await user.type(screen.getByLabelText("Name"), nameValue);
+ await user.type(screen.getByLabelText("Description"), descriptionValue);
+ await user.click(screen.getByRole("button", { name: "Save" }));
+
+ expect(onSave).toHaveBeenCalledWith(
+ { name: nameValue, description: descriptionValue },
+ true
+ );
+ });
+});
diff --git a/frontend/pages/labels/components/ManualLabelForm/ManualLabelForm.tests.tsx b/frontend/pages/labels/components/ManualLabelForm/ManualLabelForm.tests.tsx
new file mode 100644
index 0000000000..04b3a84ee3
--- /dev/null
+++ b/frontend/pages/labels/components/ManualLabelForm/ManualLabelForm.tests.tsx
@@ -0,0 +1,49 @@
+import React from "react";
+import { noop } from "lodash";
+import { screen } from "@testing-library/react";
+
+import { createCustomRenderer } from "test/test-utils";
+import createMockHost from "__mocks__/hostMock";
+
+import ManualLabelForm, {
+ LABEL_TARGET_HOSTS_INPUT_LABEL,
+} from "./ManualLabelForm";
+
+describe("ManualLabelForm", () => {
+ it("should render a Select Hosts input", () => {
+ const render = createCustomRenderer({ withBackendMock: true });
+
+ render();
+
+ expect(
+ screen.getByText(LABEL_TARGET_HOSTS_INPUT_LABEL)
+ ).toBeInTheDocument();
+ });
+
+ it("should pass up the form data when the form is submitted and valid", async () => {
+ const render = createCustomRenderer({ withBackendMock: true });
+ const onSave = jest.fn();
+
+ const name = "Test Name";
+ const description = "Test Description";
+ const targetedHosts = [createMockHost()];
+
+ const { user } = render(
+
+ );
+
+ await user.type(screen.getByLabelText("Name"), name);
+ await user.type(screen.getByLabelText("Description"), description);
+ await user.click(screen.getByRole("button", { name: "Save" }));
+
+ expect(onSave).toHaveBeenCalledWith({
+ name,
+ description,
+ targetedHosts,
+ });
+ });
+});
diff --git a/frontend/pages/labels/components/ManualLabelForm/ManualLabelForm.tsx b/frontend/pages/labels/components/ManualLabelForm/ManualLabelForm.tsx
index 77b243a337..c4f52af7b6 100644
--- a/frontend/pages/labels/components/ManualLabelForm/ManualLabelForm.tsx
+++ b/frontend/pages/labels/components/ManualLabelForm/ManualLabelForm.tsx
@@ -14,7 +14,7 @@ import { generateTableHeaders } from "./LabelHostTargetTableConfig";
const baseClass = "ManualLabelForm";
-const LABEL_TARGET_HOSTS_INPUT_LABEL = "Select hosts";
+export const LABEL_TARGET_HOSTS_INPUT_LABEL = "Select hosts";
const LABEL_TARGET_HOSTS_INPUT_PLACEHOLDER =
"Search name, hostname, or serial number";
const DEBOUNCE_DELAY = 500;
diff --git a/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx b/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx
index 2f848e5f13..14b7cc7b4d 100644
--- a/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx
+++ b/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx
@@ -794,7 +794,7 @@ const ManagePolicyPage = ({
value: "calendar_events",
disabled: !isPremiumTier || isAllTeams,
helpText: "Automatically reserve time to resolve failing policies.",
- disabledTooltipContent,
+ tooltipContent: disabledTooltipContent,
},
{
label: "Other workflows",
diff --git a/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx b/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx
index 7e3285d87d..9ffbd5fa55 100644
--- a/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx
+++ b/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx
@@ -75,7 +75,7 @@ const CalendarEventsModal = ({
const { url: newUrl } = newFormData;
if (
formData.enabled &&
- !validURL({ url: newUrl || "", protocol: "http" })
+ !validURL({ url: newUrl || "", protocols: ["http", "https"] })
) {
const errorPrefix = newUrl ? `${newUrl} is not` : "Please enter";
errors.url = `${errorPrefix} a valid resolution webhook URL`;
diff --git a/frontend/pages/policies/constants.ts b/frontend/pages/policies/constants.ts
index 6bc9ded8a3..a4284615a2 100644
--- a/frontend/pages/policies/constants.ts
+++ b/frontend/pages/policies/constants.ts
@@ -126,7 +126,7 @@ export const DEFAULT_POLICIES: IPolicyNew[] = [
query: "SELECT 1 FROM mdm WHERE enrolled='true';",
name: "MDM enrolled (macOS)",
description:
- "Required: osquery deployed with Orbit, or manual installation of macadmins/osquery-extension. Checks that a Mac is enrolled to MDM. Add a AND on identity_certificate_uuid to check for a specific MDM.",
+ "Checks that a Mac is enrolled to MDM. Add a AND on identity_certificate_uuid to check for a specific MDM.",
resolution: "Enroll device to MDM",
critical: false,
platform: "darwin",
diff --git a/frontend/pages/queries/edit/EditQueryPage.tsx b/frontend/pages/queries/edit/EditQueryPage.tsx
index b2b04a136a..1c105e6220 100644
--- a/frontend/pages/queries/edit/EditQueryPage.tsx
+++ b/frontend/pages/queries/edit/EditQueryPage.tsx
@@ -5,7 +5,12 @@ import { InjectedRouter, Params } from "react-router/lib/Router";
import { AppContext } from "context/app";
import { QueryContext } from "context/query";
-import { DEFAULT_QUERY, DOCUMENT_TITLE_SUFFIX } from "utilities/constants";
+import {
+ DEFAULT_QUERY,
+ DOCUMENT_TITLE_SUFFIX,
+ INVALID_PLATFORMS_FLASH_MESSAGE,
+ INVALID_PLATFORMS_REASON,
+} from "utilities/constants";
import configAPI from "services/entities/config";
import queryAPI from "services/entities/queries";
import statusAPI from "services/entities/status";
@@ -15,6 +20,7 @@ import {
ISchedulableQuery,
} from "interfaces/schedulable_query";
import { IConfig } from "interfaces/config";
+import { getErrorReason } from "interfaces/errors";
import QuerySidePanel from "components/side_panels/QuerySidePanel";
import MainContent from "components/MainContent";
@@ -229,7 +235,7 @@ const EditQueryPage = ({
renderFlash("success", "Query created!");
setBackendValidators({});
} catch (createError: any) {
- if (createError.data.errors[0].reason.includes("already exists")) {
+ if (getErrorReason(createError).includes("already exists")) {
const teamErrorText =
teamNameForQuery && apiTeamIdForQuery !== 0
? `the ${teamNameForQuery} team`
@@ -275,8 +281,11 @@ const EditQueryPage = ({
refetchStoredQuery(); // Required to compare recently saved query to a subsequent save to the query
} catch (updateError: any) {
console.error(updateError);
- if (updateError.data.errors[0].reason.includes("Duplicate")) {
+ const reason = getErrorReason(updateError);
+ if (reason.includes("Duplicate")) {
renderFlash("error", "A query with this name already exists.");
+ } else if (reason.includes(INVALID_PLATFORMS_REASON)) {
+ renderFlash("error", INVALID_PLATFORMS_FLASH_MESSAGE);
} else {
renderFlash(
"error",
diff --git a/frontend/pages/queries/edit/components/EditQueryForm/EditQueryForm.tsx b/frontend/pages/queries/edit/components/EditQueryForm/EditQueryForm.tsx
index 512fe6d755..98fc461e3c 100644
--- a/frontend/pages/queries/edit/components/EditQueryForm/EditQueryForm.tsx
+++ b/frontend/pages/queries/edit/components/EditQueryForm/EditQueryForm.tsx
@@ -27,9 +27,11 @@ import {
SCHEDULE_PLATFORM_DROPDOWN_OPTIONS,
MIN_OSQUERY_VERSION_OPTIONS,
LOGGING_TYPE_OPTIONS,
+ INVALID_PLATFORMS_REASON,
+ INVALID_PLATFORMS_FLASH_MESSAGE,
} from "utilities/constants";
import usePlatformCompatibility from "hooks/usePlatformCompatibility";
-import { IApiError } from "interfaces/errors";
+import { getErrorReason, IApiError } from "interfaces/errors";
import {
ISchedulableQuery,
ICreateQueryRequestBody,
@@ -328,7 +330,8 @@ const EditQueryForm = ({
renderFlash("success", `Successfully added query.`);
})
.catch((createError: { data: IApiError }) => {
- if (createError.data.errors[0].reason.includes("already exists")) {
+ const createErrorReason = getErrorReason(createError);
+ if (createErrorReason.includes("already exists")) {
queryAPI
.create({
name: `Copy of ${lastEditedQueryName}`,
@@ -351,9 +354,7 @@ const EditQueryForm = ({
})
.catch((createCopyError: { data: IApiError }) => {
if (
- createCopyError.data.errors[0].reason.includes(
- "already exists"
- )
+ getErrorReason(createCopyError).includes("already exists")
) {
let teamErrorText;
if (apiTeamIdForQuery !== 0) {
@@ -372,6 +373,9 @@ const EditQueryForm = ({
}
setIsSaveAsNewLoading(false);
});
+ } else if (createErrorReason.includes(INVALID_PLATFORMS_REASON)) {
+ setIsSaveAsNewLoading(false);
+ renderFlash("error", INVALID_PLATFORMS_FLASH_MESSAGE);
} else {
setIsSaveAsNewLoading(false);
renderFlash("error", "Could not create query. Please try again.");
diff --git a/frontend/test/handlers/label-handlers.ts b/frontend/test/handlers/label-handlers.ts
new file mode 100644
index 0000000000..da1c8959c2
--- /dev/null
+++ b/frontend/test/handlers/label-handlers.ts
@@ -0,0 +1,15 @@
+import { rest } from "msw";
+
+import { baseUrl } from "test/test-utils";
+import { createMockLabel } from "__mocks__/labelsMock";
+import { ILabel } from "interfaces/label";
+
+// eslint-disable-next-line import/prefer-default-export
+export const getLabelHandler = (overrides: Partial) =>
+ rest.get(baseUrl("/labels/:id"), (req, res, context) => {
+ return res(
+ context.json({
+ label: createMockLabel({ ...overrides }),
+ })
+ );
+ });
diff --git a/frontend/test/test-utils.tsx b/frontend/test/test-utils.tsx
index 878b08d8ea..4111300afc 100644
--- a/frontend/test/test-utils.tsx
+++ b/frontend/test/test-utils.tsx
@@ -47,15 +47,6 @@ interface ICustomRenderOptions {
withBackendMock?: boolean;
}
-// TODO: types
-// type RenderOptionsWithoutUserEvents = ICustomRenderOptions & {
-// withUserEvents: false;
-// };
-
-// type RenderOptionsWithUserEvents = ICustomRenderOptions & {
-// withUserEvents: true;
-// };
-
const CONTEXT_PROVIDER_MAP = {
app: AppContext,
notification: NotificationContext,
diff --git a/frontend/utilities/constants.tsx b/frontend/utilities/constants.tsx
index e16a487145..b3aaca9503 100644
--- a/frontend/utilities/constants.tsx
+++ b/frontend/utilities/constants.tsx
@@ -26,6 +26,12 @@ export const DEFAULT_GRAVATAR_LINK_FALLBACK =
export const DEFAULT_GRAVATAR_LINK_DARK_FALLBACK =
"/assets/images/icon-avatar-default-dark-24x24%402x.png";
+export const ACTIVITY_EXPIRY_WINDOW_DROPDOWN_OPTIONS: IDropdownOption[] = [
+ { value: 30, label: "30 days" },
+ { value: 60, label: "60 days" },
+ { value: 90, label: "90 days" },
+];
+
export const FREQUENCY_DROPDOWN_OPTIONS: IDropdownOption[] = [
{ value: 0, label: "Never" },
{ value: 300, label: "Every 5 minutes" },
@@ -424,3 +430,9 @@ export const DEFAULT_USE_QUERY_OPTIONS = {
retry: 3,
refetchOnWindowFocus: false,
};
+
+export const INVALID_PLATFORMS_REASON =
+ "query payload verification: query's platform must be a comma-separated list of 'darwin', 'linux', 'windows', and/or 'chrome' in a single string";
+
+export const INVALID_PLATFORMS_FLASH_MESSAGE =
+ "Couldn't save query. Please update platforms and try again.";
diff --git a/go.mod b/go.mod
index 4aa773996d..c7b6e28eb8 100644
--- a/go.mod
+++ b/go.mod
@@ -30,10 +30,11 @@ require (
github.com/doug-martin/goqu/v9 v9.18.0
github.com/e-dard/netbug v0.0.0-20151029172837-e64d308a0b20
github.com/elazarl/go-bindata-assetfs v1.0.1
- github.com/facebookincubator/nvdtools v0.1.6-0.20231010102659-d14ce526f176
+ github.com/facebookincubator/flog v0.0.0-20190930132826-d2511d0ce33c
github.com/fatih/color v1.15.0
github.com/getsentry/sentry-go v0.18.0
github.com/ghodss/yaml v1.0.0
+ github.com/github/smimesign v0.2.0
github.com/go-git/go-git/v5 v5.11.0
github.com/go-ini/ini v1.67.0
github.com/go-kit/kit v0.12.0
@@ -58,14 +59,14 @@ require (
github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95
github.com/hillu/go-ntdll v0.0.0-20220801201350-0d23f057ef1f
github.com/igm/sockjs-go/v3 v3.0.2
- github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5
+ github.com/jmoiron/sqlx v1.3.5
github.com/josephspurrier/goversioninfo v1.4.0
github.com/kevinburke/go-bindata v3.24.0+incompatible
github.com/kolide/launcher v1.0.12
github.com/lib/pq v1.10.9
github.com/macadmins/osquery-extension v0.0.15
github.com/mattermost/xml-roundtrip-validator v0.0.0-20201213122252-bcd7e1b9601e
- github.com/mattn/go-sqlite3 v1.14.13
+ github.com/mattn/go-sqlite3 v1.14.22
github.com/micromdm/micromdm v1.9.0
github.com/mitchellh/go-ps v1.0.0
github.com/mitchellh/gon v0.2.6-0.20231031204852-2d4f161ccecd
@@ -82,7 +83,7 @@ require (
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pkg/errors v0.9.1
github.com/pmezard/go-difflib v1.0.0
- github.com/prometheus/client_golang v1.13.0
+ github.com/prometheus/client_golang v1.19.0
github.com/quasilyte/go-ruleguard/dsl v0.3.22
github.com/rs/zerolog v1.20.0
github.com/russellhaering/goxmldsig v1.2.0
@@ -115,7 +116,7 @@ require (
golang.org/x/image v0.10.0
golang.org/x/mod v0.12.0
golang.org/x/net v0.24.0
- golang.org/x/oauth2 v0.12.0
+ golang.org/x/oauth2 v0.16.0
golang.org/x/sync v0.3.0
golang.org/x/sys v0.19.0
golang.org/x/text v0.14.0
@@ -206,7 +207,6 @@ require (
github.com/elastic/go-sysinfo v1.7.1 // indirect
github.com/elastic/go-windows v1.0.1 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
- github.com/facebookincubator/flog v0.0.0-20190930132826-d2511d0ce33c // indirect
github.com/fatih/structs v1.1.0 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
@@ -259,7 +259,6 @@ require (
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/mattn/go-tty v0.0.3 // indirect
- github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.4.3 // indirect
@@ -271,9 +270,9 @@ require (
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/term v0.0.0-20190109203006-aa71e9d9e942 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
- github.com/prometheus/client_model v0.2.0 // indirect
- github.com/prometheus/common v0.37.0 // indirect
- github.com/prometheus/procfs v0.8.0 // indirect
+ github.com/prometheus/client_model v0.5.0 // indirect
+ github.com/prometheus/common v0.48.0 // indirect
+ github.com/prometheus/procfs v0.12.0 // indirect
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/saferwall/pe v1.5.2 // indirect
diff --git a/go.sum b/go.sum
index 8dd5d62aee..3763eb862c 100644
--- a/go.sum
+++ b/go.sum
@@ -31,10 +31,6 @@ cloud.google.com/go v0.92.2/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y
cloud.google.com/go v0.92.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI=
cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI=
cloud.google.com/go v0.94.0/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4=
-cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4=
-cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc=
-cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM=
-cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA=
cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME=
cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
@@ -51,7 +47,6 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
cloud.google.com/go/firestore v1.5.0/go.mod h1:c4nNYR1qdq7eaZ+jSc5fonrQN2k3M7sWATcYTiakjEo=
-cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY=
cloud.google.com/go/iam v1.1.2 h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4=
cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU=
cloud.google.com/go/kms v0.1.0/go.mod h1:8Qp8PCAypHg4FdmlyW1QRAv09BGQ9Uzh7JnmIZxPk+c=
@@ -141,13 +136,11 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ
github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
-github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
@@ -204,10 +197,7 @@ github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/g
github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 h1:NjwIgLQlD46o79bheVG4SCdRnnOz4XtgUN1WABX5DLA=
github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
@@ -236,7 +226,6 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdK
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
-github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
@@ -313,17 +302,15 @@ github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqy
github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
-github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I=
github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -337,9 +324,7 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP
github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -351,7 +336,6 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -380,7 +364,6 @@ github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KP
github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI=
github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
@@ -430,16 +413,10 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
-github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws=
github.com/facebookincubator/flog v0.0.0-20190930132826-d2511d0ce33c h1:KqlxcP2nuOcMjudCvK0qME2K/aFBDH+xcvYv7HYQaYc=
github.com/facebookincubator/flog v0.0.0-20190930132826-d2511d0ce33c/go.mod h1:QGzNH9ujQ2ZUr/CjDGZGWeDAVStrWNjHeEcjJL96Nuk=
-github.com/facebookincubator/nvdtools v0.1.6-0.20231010102659-d14ce526f176 h1:a8y0ludOtb3gZFy8SHcy6xgKEujEd/GeNO1FicC9frg=
-github.com/facebookincubator/nvdtools v0.1.6-0.20231010102659-d14ce526f176/go.mod h1:Kh55SAWnjckS96TBSrXI99KrEKH4iB0OJby3N8GRJO4=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
-github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
@@ -463,6 +440,8 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
+github.com/github/smimesign v0.2.0 h1:Hho4YcX5N1I9XNqhq0fNx0Sts8MhLonHd+HRXVGNjvk=
+github.com/github/smimesign v0.2.0/go.mod h1:iZiiwNT4HbtGRVqCQu7uJPEZCuEE5sfSSttcnePkDl4=
github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
@@ -488,16 +467,12 @@ github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-kit/kit v0.7.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.12.0 h1:e4o3o3IsBfAKQh5Qbbiqyfu97Ku7jrO/JbohvztANh4=
github.com/go-kit/kit v0.12.0/go.mod h1:lHd+EkCZPIwYItmGDDRdhinkzX2A1sj+M9biaEaizzs=
-github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
-github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -515,7 +490,6 @@ github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTM
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
-github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
@@ -672,7 +646,6 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
-github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -707,33 +680,25 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 h1:RtRsiaGvWxcwd8y3BiRZxsylPT8hLWZ5SPcfI+3IDNk=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0/go.mod h1:TzP6duP4Py2pHLVPPQp42aoYI92+PCrVotyR5e8Vqlk=
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
-github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
-github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
github.com/hashicorp/go-hclog v0.9.3-0.20191025211905-234833755cb2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
-github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
github.com/hashicorp/go-hclog v1.0.0 h1:bkKf0BeBXcSYa7f5Fyi9gMuQ8gNsxeiNpZjR6VxNZeo=
github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
-github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs=
github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
-github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
@@ -750,14 +715,8 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T
github.com/hashicorp/hcl/v2 v2.0.0/go.mod h1:oVVDG71tEinNGYCxinCYadcmKU9bglqW9pV3txagJ90=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
-github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY=
-github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc=
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
-github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
-github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
-github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
-github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ=
github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E=
github.com/hillu/go-ntdll v0.0.0-20220801201350-0d23f057ef1f h1:es0IoL1/OOoGYUuvRtSzbtG3STd7Fm5LIniUWsfzMHE=
@@ -768,7 +727,6 @@ github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
github.com/iancoleman/orderedmap v0.2.0 h1:sq1N/TFpYH++aViPcaKjys3bDClUEU7s5B+z6jq8pNA=
github.com/iancoleman/orderedmap v0.2.0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
-github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/igm/sockjs-go/v3 v3.0.2 h1:2m0k53w0DBiGozeQUIEPR6snZFmpFpYvVsGnfLPNXbE=
@@ -792,8 +750,8 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jmoiron/sqlx v0.0.0-20180406164412-2aeb6a910c2b/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU=
-github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5 h1:lrdPtrORjGv1HbbEvKWDUAy97mPpFm4B8hp77tcCUJY=
-github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
+github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4=
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
@@ -803,17 +761,12 @@ github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUB
github.com/josephspurrier/goversioninfo v1.4.0 h1:Puhl12NSHUSALHSuzYwPYQkqa2E1+7SrtAPJorKK0C8=
github.com/josephspurrier/goversioninfo v1.4.0/go.mod h1:JWzv5rKQr+MmW+LvM412ToT/IkYDZjaclF2pKDss8IY=
github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
-github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kevinburke/go-bindata v3.24.0+incompatible h1:qajFA3D0pH94OTLU4zcCCKCDgR+Zr2cZK/RPJHDdFoY=
github.com/kevinburke/go-bindata v3.24.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM=
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
@@ -832,7 +785,6 @@ github.com/kolide/kit v0.0.0-20221107170827-fb85e3d59eab/go.mod h1:OYYulo9tUqRad
github.com/kolide/launcher v1.0.12 h1:f2uT1kKYGIbj/WVsHDc10f7MIiwu8MpmgwaGaT7D09k=
github.com/kolide/launcher v1.0.12/go.mod h1:j854Q4LqMXi3DQ+fnDy8Ij4uuKRG707ulWOcIz7BCz4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -848,13 +800,13 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.10.1/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
-github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
github.com/macadmins/osquery-extension v0.0.15 h1:uixbimhzKZSguAcLwKAfi0fieB7gIkxm3saPl9mNl9c=
github.com/macadmins/osquery-extension v0.0.15/go.mod h1:gLiR0LcxYjx71EEg70gzV7ah2skWuLw3hwR4eiV+VSw=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -870,9 +822,6 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI=
@@ -881,35 +830,28 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
-github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-github.com/mattn/go-sqlite3 v1.14.13 h1:1tj15ngiFfcZzii7yd82foL+ks+ouQcj8j/TPq3fk1I=
-github.com/mattn/go-sqlite3 v1.14.13/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mattn/go-tty v0.0.3 h1:5OfyWorkyO7xP52Mq7tB36ajHDG5OHrmBGIS/DtakQI=
github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
-github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/micromdm/micromdm v1.9.0 h1:FAsIKOpnGcq21UQCrHCUxZwSW4NwBLGOoUtzbURxds8=
github.com/micromdm/micromdm v1.9.0/go.mod h1:YsAtsEvfEIwpjYTUPpWkJXSfH0hhp9mMHW1BgIZgRt8=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
-github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
-github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
@@ -942,11 +884,9 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/ngrok/sqlmw v0.0.0-20211220175533-9d16fdc47b31 h1:FFHgfAIoAXCCL4xBoAugZVpekfGmZ/fBBueneUKBv7I=
@@ -982,9 +922,9 @@ github.com/osquery/osquery-go v0.0.0-20230603132358-d2e851b3991b/go.mod h1:OSR0O
github.com/pandatix/nvdapi v0.6.4 h1:gix57FcQtOklCUgFrJzJhRblYj+2DN9jxZP6oqtme+A=
github.com/pandatix/nvdapi v0.6.4/go.mod h1:DVYxPq0JRERgYzFmwTMknAtH4kB8v9KG+z40JWFRClk=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pborman/getopt v0.0.0-20180811024354-2b5b3bfb099b/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
@@ -1004,42 +944,27 @@ github.com/pkg/term v0.0.0-20190109203006-aa71e9d9e942/go.mod h1:eCbImbZ95eXtAUI
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
-github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
-github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
-github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
-github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU=
-github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=
+github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
+github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
-github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
-github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
-github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
-github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
-github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/quasilyte/go-ruleguard/dsl v0.3.22 h1:wd8zkOhSNr+I+8Qeciml08ivDt1pSXe60+5DqOpCjPE=
github.com/quasilyte/go-ruleguard/dsl v0.3.22/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU=
@@ -1090,8 +1015,6 @@ github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnj
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
@@ -1111,7 +1034,6 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY=
github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
@@ -1121,7 +1043,6 @@ github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA=
github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
-github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4=
github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
@@ -1180,7 +1101,6 @@ github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM=
github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
-github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
@@ -1238,9 +1158,6 @@ go.elastic.co/fastjson v1.1.0/go.mod h1:boNGISWMjQsUPy/t6yqt2/1Wx4YNPSe+mZjlyw9v
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
-go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
-go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
-go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs=
go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 h1:CCriYyAfq1Br1aIYettdHZTy8mBTIPo7We18TuO/bak=
go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=
@@ -1285,7 +1202,6 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
gocloud.dev v0.24.0 h1:cNtHD07zQQiv02OiwwDyVMuHmR7iQt2RLkzoAgz7wBs=
@@ -1300,7 +1216,6 @@ golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -1380,7 +1295,6 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -1411,18 +1325,13 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
@@ -1449,11 +1358,8 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ
golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4=
-golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4=
+golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ=
+golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1482,7 +1388,6 @@ golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1494,8 +1399,6 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1506,11 +1409,9 @@ golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1521,8 +1422,6 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1537,36 +1436,25 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210223095934-7937bea0104d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1634,7 +1522,6 @@ golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgw
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -1723,10 +1610,6 @@ google.golang.org/api v0.52.0/go.mod h1:Him/adpjt0sxtkWViy0b6xyKW/SD71CwdJ7HqJo7
google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k=
google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
-google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI=
-google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU=
-google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
-google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw=
google.golang.org/api v0.128.0 h1:RjPESny5CnQRn9V6siglged+DZCgfu9l6mO9dkX9VOg=
google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
@@ -1799,16 +1682,6 @@ google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEc
google.golang.org/genproto v0.0.0-20210825212027-de86158e7fda/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0=
google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk=
google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a h1:myvhA4is3vrit1a6NZCWBIwN0kNEnX21DJOJX/NvIfI=
@@ -1840,8 +1713,6 @@ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ
google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
@@ -1876,7 +1747,6 @@ gopkg.in/guregu/null.v3 v3.5.0 h1:xTcasT8ETfMcUHn0zTvIYtQud/9Mx5dJqD554SZct0o=
gopkg.in/guregu/null.v3 v3.5.0/go.mod h1:E4tX2Qe3h7QdL+uZ3a0vqvYwKQsRSQKM5V4YltdgH9Y=
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/mail.v2 v2.3.1 h1:WYFn/oANrAGP2C0dcV6/pbkPzv8yGzqTjPmTeO7qoXk=
@@ -1893,7 +1763,6 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/handbook/business-operations/README.md b/handbook/business-operations/README.md
index 3ff5f6d363..93fb1e69d6 100644
--- a/handbook/business-operations/README.md
+++ b/handbook/business-operations/README.md
@@ -1,12 +1,11 @@
# Business Operations
-This handbook page details processes specific to working [with](#what-we-do) and [within](#responsibilities) the Business Operations (BizOps) department.
+This handbook page details processes specific to working [with](#contact-us) and [within](#responsibilities) this department.
## Team
| Role | Contributor(s) |
|:------------------------------|:-----------------------------------------------------------------------------------------------------------|
| Head of Business Operations | [Joanne Stableford](https://www.linkedin.com/in/joanne-stableford/) _([@jostableford](https://github.com/JoStableford))_
-| Community Advocate | [JD Strong](https://www.linkedin.com/in/jackdaniyelstrong/) _([@spokanemac](https://github.com/spokanemac/spokanemac))_
-| Business Operations Engineer | [Nathan Holliday](https://www.linkedin.com/in/nathanael-holliday/) _([@hollidayn](https://github.com/hollidayn))_, [Isabell Reedy](https://www.linkedin.com/in/isabell-reedy-202aa3123/) _([@ireedy](https://github.com/ireedy))_
+| Business Operations Engineer | [Nathan Holliday](https://www.linkedin.com/in/nathanael-holliday/) _([@hollidayn](https://github.com/hollidayn))_
[Isabell Reedy](https://www.linkedin.com/in/isabell-reedy-202aa3123/) _([@ireedy](https://github.com/ireedy))_
## Contact us
- To **make a request** of this department, [create an issue](https://github.com/fleetdm/confidential/issues/new?assignees=&labels=%23g-business-operations&projects=&template=custom-request.md&title=Request%3A+_______________________) and a team member will get back to you within one business day (If urgent, mention a [team member](#team) in [#g-business-operations](https://fleetdm.slack.com/archives/C047N5L6EGH).
diff --git a/handbook/business-operations/security-audits.md b/handbook/business-operations/security-audits.md
index b9628c4b2c..3be3f10609 100644
--- a/handbook/business-operations/security-audits.md
+++ b/handbook/business-operations/security-audits.md
@@ -2,7 +2,30 @@
This page contains explanations of the latest external security audits performed on Fleet software.
## June 2023 penetration testing of Fleet 4.33
-Available on request.
+In June 2023, [Latacora](https://www.latacora.com/) performed an application penetration assessment of the application from Fleet.
+
+An application penetration test captures a point-in-time assessment of vulnerabilities, misconfigurations, and gaps in applications that could allow an attacker to compromise the security, availability, processing integrity, confidentiality, and privacy (SAPCP) of sensitive data and application resources. An application penetration test simulates the capabilities of a real adversary, but accelerates testing by using information provided by the target company.
+
+You can find the full report here: [2023-06-09-fleet-penetration-test.pdf](https://github.com/fleetdm/fleet/raw/main/docs/files/2023-06-09-fleet-penetration-test.pdf).
+
+### Findings
+#### 1 - Stored cross-site scripting (XSS) in tooltip
+| Type | Latacora Severity |
+| ------------------- | -------------- |
+| Cross-site scripting| High risk |
+
+All tooltips using the "tipContent" tag are set using "dangerouslySetInnerHTML". This allows manipulation of the DOM without sanitization. If a user can control the content sent to this function, it can lead to a cross-site scripting vulnerability.
+
+- Resolved. Resolution information TBA
+
+#### 2 - Broken authorization leads to observers able to add hosts
+| Type | Latacora Severity |
+| ------------------- | -------------- |
+| Authorization issue | High risk |
+
+Observers are not supposed to be able to add hosts to Fleet. Via specific endpoints, it becomes possible to retrieve the certificate chains and the secrets for all teams, and these are the information required to add a host.
+
+- Resolved. Resolution information TBA
## April 2022 penetration testing of Fleet 4.12
In April 2022, we worked with [Lares](https://www.lares.com/) to perform penetration testing on our Fleet instance, which was running 4.12 at the time.
diff --git a/handbook/company/README.md b/handbook/company/README.md
index 05bbe2f238..ee0874d669 100644
--- a/handbook/company/README.md
+++ b/handbook/company/README.md
@@ -135,7 +135,7 @@ Fleet added support for [scripting and management capabilities](https://fleetdm.
> Still curious? Check out this [visualization of the Fleet repo over the years](https://www.linkedin.com/feed/update/urn:li:activity:7045068060168220672/) or listen to this [conversation between Zach and Mike Arpaia about the origin story of osquery](https://fleetdm.com/podcasts/the-future-of-device-management-ep1).
## Org chart
-To provide clarity about decision-making, [responsibility](https://fleetdm.com/handbook/company/why-this-way#why-direct-responsibility), and resources, everyone at Fleet has a manager, and [every manager](https://fleetdm.com/handbook/company#management) has direct reports. Fleet's organizational chart is accessible company-wide as a sub-tab in ["🧑🚀 Fleeties" (private google doc)](https://docs.google.com/spreadsheets/d/1OSLn-ZCbGSjPusHPiR5dwQhheH1K8-xqyZdsOe9y7qc/edit#gid=0). On the other sub-tabs, you can also check out a world map of where everyone is located, hiring stats, and fun facts about each team member.
+To provide clarity about decision-making, [responsibility](https://fleetdm.com/handbook/company/why-this-way#why-direct-responsibility), and resources, everyone at Fleet has a manager, and [every manager](https://fleetdm.com/handbook/company/leadership) has direct reports. Fleet's organizational chart is accessible company-wide as a sub-tab in ["🧑🚀 Fleeties" (private google doc)](https://docs.google.com/spreadsheets/d/1OSLn-ZCbGSjPusHPiR5dwQhheH1K8-xqyZdsOe9y7qc/edit#gid=0). On the other sub-tabs, you can also check out a world map of where everyone is located, hiring stats, and fun facts about each team member.
- 🔦 [Business Operations](https://fleetdm.com/handbook/business-operations): The Business Operations department is directly responsible for these traditional functions: People, Finance, tax, compliance, Legal, and IT.
- 🌦️ [Customer Success](https://fleetdm.com/handbook/customer-success): The customer success department is directly responsible for ensuring that customers and community members of Fleet achieve their desired outcomes with Fleet products and services.
diff --git a/handbook/company/communications.md b/handbook/company/communications.md
index 5df2104ffa..bb46f45a2a 100644
--- a/handbook/company/communications.md
+++ b/handbook/company/communications.md
@@ -13,19 +13,41 @@ You can read about the company's positioning and product strategy in ["🎐 Why
We track competitors' capabilities and adjacent (or commonly integrated) products in Google doc [Competition](https://docs.google.com/document/d/1Bqdui6oQthdv5XtD5l7EZVB-duNRcqVRg7NVA4lCXeI/edit) (private Google doc).
## Directly responsible individuals (DRIs)
-| Responsibility | DRI |
-| -------------- | --- |
-| Intentionality of Fleet's interfaces | [Noah Talerman](https://www.linkedin.com/in/noah-talerman/) _([@noahtalerman](https://github.com/noahtalerman))_ |
-| Best practices for using Fleet | [Noah Talerman](https://www.linkedin.com/in/noah-talerman/) _([@noahtalerman](https://github.com/noahtalerman))_ |
-| What goes in a release | [Luke Heath](https://www.linkedin.com/in/lukeheath/) _([@lukeheath](https://github.com/lukeheath))_ |
-| Engineering output and architecture | [Luke Heath](https://www.linkedin.com/in/lukeheath/) _([@lukeheath](https://github.com/lukeheath))_ |
-| Structure and intentionality of the [Docs](https://fleetdm.com/docs/get-started/why-fleet)| [Mike Thomas](https://www.linkedin.com/in/mike-thomas-52277938) _([@mike-j-thomas](https://github.com/mike-j-thomas))_ |
-| Design and content of the [Docs](https://fleetdm.com/docs/get-started/why-fleet) | [Rachael Shaw](https://www.linkedin.com/in/rachaelcshaw/) _([@rachaelshaw](https://github.com/rachaelshaw))_ |
-| API design | [Rachael Shaw](https://www.linkedin.com/in/rachaelcshaw/) _([@rachaelshaw](https://github.com/rachaelshaw))_ |
+| Area of responsibility | [DRI](https://fleetdm.com/handbook/company/why-this-way#why-direct-responsibility) |
+| -------------- | --- |
+| Revenue | _See [🐋 Chief Revenue Officer](https://fleetdm.com/handbook/sales#team)_
+| Pricing | _See [🛠️ CEO responsibilities](https://fleetdm.com/handbook/company/leadership#ceo-responsibilities)_
+| Illustrations | _See [🌐 Head of Design](https://fleetdm.com/handbook/digital-experience#team)_
+| Website | _See [🌐 Digital Experience team](https://fleetdm.com/handbook/digital-experience#team)_
+| Product marketing (PMM) | _See [🛠️ CEO responsibilities](https://fleetdm.com/handbook/company/leadership#ceo-responsibilities)_
+| Brand marketing | _See [🛠️ CEO responsibilities](https://fleetdm.com/handbook/company/leadership#ceo-responsibilities)_
+| Public relations | _See [🛠️ CEO responsibilities](https://fleetdm.com/handbook/company/leadership#ceo-responsibilities)_
+| Revenue pipeline | _See [🫧 Head of Demand Generation](https://fleetdm.com/handbook/demand#team)_
+| Ads | _See [🫧 Demand team](https://fleetdm.com/handbook/demand#team)_
+| Video | _See [🫧 Digital Marketing Manager](https://fleetdm.com/handbook/demand#team)_
+| Social media | _See [🫧 Digital Marketing Manager](https://fleetdm.com/handbook/demand#team)_
+| Blog | _See [🚀 Client Platform Engineer & Community Advocate](https://fleetdm.com/handbook/engineering#team)_
+| Information technology (IT) | _See [🚀 Client Platform Engineer & Community Advocate](https://fleetdm.com/handbook/engineering#team)_
+| Payroll, bookkeeping, AR/AP | _See [🔦 Head of Business Operations](https://fleetdm.com/handbook/customer-success#team)_
+| Legal contracts | _See [🔦 Business Operations team](https://fleetdm.com/handbook/customer-success#team)_
+| Customer renewals | _See [🌦️ VP of Customer Success](https://fleetdm.com/handbook/customer-success#team)_
+| Customer deployments | _See [🌦️ Infrastructure Engineer](https://fleetdm.com/handbook/customer-success#team)_
+| Customer support | _See [🌦️ Customer Success team](https://fleetdm.com/handbook/customer-success#team)_
+| Quality assurance (QA) | _See [🚀 Engineering team](https://fleetdm.com/handbook/engineering#team)_
+| Features & product adoption | _See [🦢 Head of Product Design](https://fleetdm.com/handbook/product-design#team)_
+| Feature prioritization | _See [🦢 Head of Product Design](https://fleetdm.com/handbook/product-design#team)_
+| Intentionality of Fleet's interfaces | _See [🦢 Head of Product Design](https://fleetdm.com/handbook/product-design#team)_
+| Best practices for using Fleet | _See [🦢 Product Design team](https://fleetdm.com/handbook/product-design#team)_
+| [API design](https://fleetdm.com/docs/rest-api/rest-api) | _See [🦢 Rachael Shaw](https://fleetdm.com/handbook/product-design#team)_
+| Structure of the [docs](https://fleetdm.com/docs/get-started/why-fleet) | _See [🌐 Head of Design](https://fleetdm.com/handbook/digital-experience#team)_
+| Product reference documentation | _See [🦢 Rachael Shaw](https://fleetdm.com/handbook/product-design#team)_
+| What goes in a release | _See [🚀 Chief Technology Officer](https://fleetdm.com/handbook/engineering#team)_
+| Engineering output and architecture | _See [🚀 Chief Technology Officer](https://fleetdm.com/handbook/engineering#team)_
+| Product development | _See [🛩️ Product groups](https://fleetdm.com/handbook/company/product-groups#current-product-groups)_
### Docs
-This page details processes related to maintaining and updating the ([Fleet docs](https://fleetdm.com/docs)).
+This page details processes related to maintaining and updating the [Fleet documentation](https://fleetdm.com/docs).
When someone asks a question in a public channel, it's safe to assume they aren't the only person looking for an answer.
@@ -33,6 +55,7 @@ To make our docs as helpful as possible, the Community team gathers these questi
Fleet's goal is to answer every question with a link to the docs and/or result in a documentation update.
+> Fleet's philosophy on how to write useful documentation is public and open-source: ["Why read documentation?"](https://fleetdm.com/handbook/company/why-this-way#why-read-documentation)
## Fleetdm.com
Any change to fleetdm.com follows the same process as [making changes](https://fleetdm.com/handbook/company/product-groups#making-changes) to the core product. To propose a change to Fleet's website [create a website request](https://github.com/fleetdm/fleet/issues/new?assignees=&labels=%23g-digital-experience&projects=&template=website-request.md&title=Request%3A+__________________________) on the #g-digital-experience kanban board.
diff --git a/handbook/company/open-positions.yml b/handbook/company/open-positions.yml
index cd91548db0..7d313763f6 100644
--- a/handbook/company/open-positions.yml
+++ b/handbook/company/open-positions.yml
@@ -61,32 +61,5 @@
- 🛠️ Technical: You understand the software development processes.
- 🟣 Openness: You are flexible and open to new ideas and ways of working
- ➕ Bonus: Cybersecurity or IT background
-- jobTitle: 🐋 Customer Success Engineer
- department: Customers
- hiringManagerName: Jason Lewis
- hiringManagerGithubUsername: Patagonia121
- hiringManagerLinkedInUrl: https://www.linkedin.com/in/jlewis0451/
- responsibilities: |
- - 🎯 Strong attention to detail and can act as an encyclopedia of knowledge about how Fleet works - our customers represent a wide range of needs across many different use cases. Be adaptable to learning new things quickly and then share this knowledge with others.
- - 📣 Manage multiple customer deployments and escalations simultaneously with the ability to stay organized.
- - 🚀 Deploy Fleet on your own to have a better understanding of the customer experience and how the product works.
- - 🪴 Promote product adoption, referencability, and customer advocacy with key customer stakeholders.
- - 🥇 Be the first line of defense in customer Slack channels for any reported problems, how-to questions, feature request intake, and bug report filling.
- - 🚀 Work collaboratively with product and engineering teams to facilitate bug resolution and feature development based on customer asks.
- - ⏫ Work hand-in-hand with the customer success team by participating in ad-hoc calls with customers to discuss any support issues they may have.
- - 💡 Excellent communication and collaboration skills, with the ability to work cross-functionally with CS, engineering, and product teams.
- experience: |
- - 💭 Cybersecurity or IT background, experience with cloud environments like AWS and Azure or device management solutions like Fleet, Intune, Jamf Pro, Workspace One, etc.
- - 💖 You know how to manage your time and priorities between customer support engagements, customer escalations, and other day-to-day responsibilities.
- - 🧬 An excellent understanding of macOS, Windows, Linux and core services like Autopilot, ABM/ASM, MDM, ADE, APNs, syslog, etc.
- - 🤝 You work best in a team-based environment. You are decisive with the ability to shift gears between thinking and doing.
- - 👥 A customer-centric mindset, focusing on delivering value and a positive user experience.
- - 🦉 2-3 years of work experience providing technical support to enterprise customers in the cybersecurity or device management space. Experience with executing and tracking results tied to customer escalations.
- - 🛠️ You are personable, enjoy being customer facing, and have a passion for problem solving while assisting external and internal stakeholders.
- - 🧪 Extensive experience with Slack, Google Suite, and GitHub.
- - ✍️ Familiarity with shell scripting, Python, Powershell, and using Terminal to execute commands or run scripts, and other line of business applications.
- - 🟣 Openness: Speak freely. Interrupt and be interrupted. Give pointed and respectful feedback, even when you disagree.
- - 🔴 Empathy: You should demonstrate empathy by keenly understanding and addressing customer concerns with genuine compassion.
- - ➕ Bonus: Familiarity with osquery, MySQL, GitOps workflows, Terraform, Tines/Torq and open source projects. Experience working with IT, SRE, CPE, or SecOps teams.
diff --git a/handbook/company/pricing-features-table.yml b/handbook/company/pricing-features-table.yml
index d22f054d7a..1621f3a65b 100644
--- a/handbook/company/pricing-features-table.yml
+++ b/handbook/company/pricing-features-table.yml
@@ -632,7 +632,7 @@
description: Easily configure and install SentinelOne, Crowdstrike, and other security tools.
moreInfoUrl: https://github.com/fleetdm/fleet/issues/14921
tier: Premium
- comingSoonOn: 2024-04-22 #customer-reedtimmer,customer-flacourtia
+ comingSoonOn: 2024-05-13 #customer-flacourtia
usualDepartment: IT
productCategories: [Device management]
pricingTableCategories: [Device management]
diff --git a/handbook/company/product-groups.md b/handbook/company/product-groups.md
index d85d80eeb7..fc66b775b3 100644
--- a/handbook/company/product-groups.md
+++ b/handbook/company/product-groups.md
@@ -52,7 +52,7 @@ The goal of the MDM group is to increase and exceed [Fleet's product maturity go
| Product Designer | [Marko Lisica](https://www.linkedin.com/in/markolisica/) _([@marko-lisica](https://github.com/marko-lisica))_
| Engineering Manager | [George Karr](https://www.linkedin.com/in/george-karr-4977b441/) _([@georgekarrv](https://github.com/georgekarrv))_
| Product Manager | [Noah Talerman](https://www.linkedin.com/in/noah-talerman/) _([@noahtalerman](https://github.com/@noahtalerman))_
-| Quality Assurance | [Position open](https://www.fleetdm.com/jobs/)
+| Quality Assurance | [Gabe Lopez](https://www.linkedin.com/in/gabelopez/) _([@PezHub](https://github.com/PezHub))_
| Developer | [Gabe Hernandez](https://www.linkedin.com/in/gabriel-hernandez-gh) _([@ghernandez345](https://github.com/ghernandez345))_, [Roberto Dip](https://www.linkedin.com/in/roperzh) _([@roperzh](https://github.com/roperzh))_, Sarah Gillespie _([@gillespi314](https://github.com/gillespi314))_, [Martin Angers](https://www.linkedin.com/in/martin-angers-3210305/) _([@mna](https://github.com/mna))_, [Jahziel Villasana-Espinoza](https://www.linkedin.com/in/jahziel-v/) _([@jahzielv](https://github.com/jahzielv))_, [Dante Catalfamo](https://www.linkedin.com/in/dante-catalfamo-a6330412b/) _([@dantecatalfamo](https://github.com/dantecatalfamo))_
> The [Slack channel](https://fleetdm.slack.com/archives/C03C41L5YEL), [kanban release board](https://app.zenhub.com/workspaces/-g-mdm-current-sprint-63bc507f6558550011840298/board), and [GitHub label](https://github.com/fleetdm/fleet/issues?q=is%3Aopen+is%3Aissue+label%3A%23g-mdm) for this product group is `#g-mdm`.
@@ -431,20 +431,20 @@ Bugs will be verified as fixed by QA when they are placed in the "Awaiting QA" c
## High priority user stories and bugs
All issues are treated as standard priority by default. Some issues are assigned a priority label to indicate urgency for the business.
-1. Emergency: `P0`
-- Examples: Customer outage, confirmed security vulnerability ([critical bug](https://fleetdm.com/handbook/company/product-groups#release-testing)), a new feature is needed to address an immediate business emergency.
-- Response: Immediately stop other work to swarm the issue. Work 24/7 in shifts until resolved.
-- Impact: Significant impact. May void current sprint.
+- Emergency: `P0`
+ - Examples: Customer outage, confirmed security vulnerability ([critical bug](https://fleetdm.com/handbook/company/product-groups#release-testing)), a new feature is needed to address an immediate business emergency.
+ - Response: Immediately stop other work to swarm the issue. Work 24/7 in shifts until resolved.
+ - Impact: Significant impact. May void current sprint.
-2. Critical: `P1`
-- Examples: A supported workflow is broken ([critical bug](https://fleetdm.com/handbook/company/product-groups#release-testing)), a potential security vulnerability, a new feature is required to address an immediate critical business need.
-- Response: Issue brought to next standup for estimation and immediately brought into the sprint. Necessary team members are assigned as their top priority.
-- Impact: High impact. Does not void sprint, but reduces overall velocity and requires deprioritizing other work.
+- Critical: `P1`
+ - Examples: A supported workflow is broken ([critical bug](https://fleetdm.com/handbook/company/product-groups#release-testing)), a potential security vulnerability, a new feature is required to address an immediate critical business need.
+ - Response: Issue brought to next standup for estimation and immediately brought into the sprint. Necessary team members are assigned as their top priority.
+ - Impact: High impact. Does not void sprint, but reduces overall velocity and requires deprioritizing other work.
-3. Urgent: `P2`
-- Examples: A supported workflow is not functioning as intended, a newly drafted feature has an associated urgent business need.
-- Response: Issue is prioritized at the top of the next sprint. If opporunity cost of waiting for the next sprint is too high, it may be considered for current sprint.
-- Impact: Low to medium impact. If prioritized into current sprint, may reduce overall velocity and require deprioritizing other work.
+- Urgent: `P2`
+ - Examples: A supported workflow is not functioning as intended, a newly drafted feature has an associated urgent business need.
+ - Response: Issue is prioritized at the top of the next sprint. If opporunity cost of waiting for the next sprint is too high, it may be considered for current sprint.
+ - Impact: Low to medium impact. If prioritized into current sprint, may reduce overall velocity and require deprioritizing other work.
Add as much context as possible to the issue description and assign labels to help the team understand the problem and what is driving the urgency. All issues with a `P0`, `P1`, or `P2` label should be assigned to the [DRI for what goes in a release](https://fleetdm.com/handbook/company/communications#directly-responsible-individuals-dris). For immediate action, follow up on Slack or by phone.
diff --git a/handbook/company/testimonials.yml b/handbook/company/testimonials.yml
index 227a42f16d..ed03143f9f 100644
--- a/handbook/company/testimonials.yml
+++ b/handbook/company/testimonials.yml
@@ -71,7 +71,7 @@
quoteAuthorJobTitle: Staff CPE at Stripe
productCategories: [Endpoint operations, Device management]
-
- quote: Fleet’s come a long way - to now being the top open-source osquery manager. Just in the past 6 months.
+ quote: Fleet’s come a long way - to now being the top open-source osquery manager.
quoteImageFilename: social-proof-logo-atlassian-192x32@2x.png
quoteLinkUrl: https://www.linkedin.com/in/bshak/
quoteAuthorName: Brendan Shaklovitz
diff --git a/handbook/customer-success/README.md b/handbook/customer-success/README.md
index 2f96814de7..9ffcf30914 100644
--- a/handbook/customer-success/README.md
+++ b/handbook/customer-success/README.md
@@ -5,10 +5,9 @@ This handbook page details processes specific to working [with](#contact-us) and
| Role | Contributor(s) |
|:--------------------------------------|:------------------------------------------------------------------------------------------------------------------------|
| VP of Customer Success | [Zay Hanlon](https://www.linkedin.com/in/zayhanlon/) _([@zayhanlon](https://github.com/zayhanlon))_
-| Customer Success Managers (CSM) | [Jason Lewis](https://www.linkedin.com/in/jlewis0451/) _([@patagonia121](https://github.com/patagonia121))_, [Michael Pinto](https://www.linkedin.com/in/michael-pinto-a06b4515a/) _([@pintomi1989](https://github.com/pintomi1989))_
-| Customer Solutions Architect (CSA) | [Brock Walters](https://www.linkedin.com/in/brock-walters-247a2990/) _([@nonpunctual](https://github.com/nonpunctual))_
-| Customer Support Engineer (CSE) | [Kathy Satterlee](https://www.linkedin.com/in/ksatter/) _([@ksatter](https://github.com/ksatter))_, [Grant Bilstad](https://www.linkedin.com/in/grantbilstad/) _([@Pacamaster](https://github.com/Pacamaster))_, Ben Edwards _([@edwardsb](https://github.com/edwardsb))_
| Infrastructure Engineer | [Robert Fairburn](https://www.linkedin.com/in/robert-fairburn/) _([@rfairburn](https://github.com/rfairburn))_
+| Customer Support (CSE/CSA) | [Kathy Satterlee](https://www.linkedin.com/in/ksatter/) _([@ksatter](https://github.com/ksatter))_
[Grant Bilstad](https://www.linkedin.com/in/grantbilstad/) _([@Pacamaster](https://github.com/Pacamaster))_
Ben Edwards _([@edwardsb](https://github.com/edwardsb))_
[Brock Walters](https://www.linkedin.com/in/brock-walters-247a2990/) _([@nonpunctual](https://github.com/nonpunctual))_
+| Customer Success Manager (CSM) | [Jason Lewis](https://www.linkedin.com/in/jlewis0451/) _([@patagonia121](https://github.com/patagonia121))_
[Michael Pinto](https://www.linkedin.com/in/michael-pinto-a06b4515a/) _([@pintomi1989](https://github.com/pintomi1989))_
## Contact us
- To **make a request** of this department, [create an issue](https://github.com/fleetdm/confidential/issues/new?assignees=&labels=%23g-customer-success&projects=&template=custom-request.md&title=Request%3A+_______________________) and a team member will get back to you within one business day (If urgent, mention a [team member](#team) in the [#g-customer-success](https://fleetdm.slack.com/archives/C062D0THVV1)).
diff --git a/handbook/demand/README.md b/handbook/demand/README.md
index 2360016984..85717e41fe 100644
--- a/handbook/demand/README.md
+++ b/handbook/demand/README.md
@@ -2,11 +2,10 @@
This handbook page details processes specific to working [with](#contact-us) and [within](#responsibilities) this department.
## Team
-| Role | Contributor(s)
-|:--------------------------------|:------------------------------------------------------------------------------------------------------------------------|
-| 🫧 Head of Demand Generation | [Dustin Gerdes](https://www.linkedin.com/in/dustingerdes/) _([@3kindsoffish](https://github.com/3kindsoffish))_
-| 🫧 Field Marketer | [Drew Baker](https://www.linkedin.com/in/andrew-baker-51547179/) _([@drewbakerfdm](https://github.com/drewbakerfdm))_
-| _🎐 Head of Brand & Product Marketing (CEO)_ | [Mike McNeil](https://www.linkedin.com/in/mikermcneil) _([@mikermcneil](https://github.com/mikermcneil))_
+| Role | Contributor(s)
+|:-----------------------------------|:------------------------------------------------------------------------------------------------------------------------|
+| Head of Demand Generation | [Dustin Gerdes](https://www.linkedin.com/in/dustingerdes/) _([@3kindsoffish](https://github.com/3kindsoffish))_
+| Digital Marketing Manager | [Drew Baker](https://www.linkedin.com/in/andrew-baker-51547179/) _([@drewbakerfdm](https://github.com/drewbakerfdm))_
## Contact us
@@ -14,10 +13,10 @@ This handbook page details processes specific to working [with](#contact-us) and
- Please use **issue comments and GitHub mentions** to communicate follow-ups or answer questions related to your request.
- Any Fleet team member can [view the kanban board](https://app.zenhub.com/workspaces/g-demand-64e6c8e2d35c7f001a457b7f/board?sprints=none) for this department, including pending tasks and the status of new requests.
-> To **make a request** related to **product marketing**, **press**, **brandfronts**, **pitchfronts**, **featurefronts**, **ideal customer profiles (ICPs)**, **personas**, or **targeting** [create an issue](https://github.com/fleetdm/confidential/issues/new?assignees=&labels=%23g-digital-experience&projects=&template=custom-request.md&title=Product%20marketing%20request%3A+_______________________) (If urgent, at-mention the [Head of Product Marketing](#team) in the [help-pmm-2023](https://fleetdm.slack.com/archives/C0600L1TTPY) Slack channel).
+> To **make a request** related to **product marketing**, **press**, **brandfronts**, **pitchfronts**, **featurefronts**, **ideal customer profiles (ICPs)**, **personas**, or **targeting** [create an issue](https://github.com/fleetdm/confidential/issues/new?assignees=&labels=%23g-digital-experience&projects=&template=custom-request.md&title=Product%20marketing%20request%3A+_______________________) (If urgent, at-mention the [Head of Product Marketing](#team) in the [#help-leadership](https://fleetdm.slack.com/archives/C0600L1TTPY) Slack channel).
## Responsibilities
-The Demand department is directly responsible for growing awareness of Fleet and nurturing the community through participation in events, conversations, and other [programs](https://fleetdm.com/handbook/company/communications#programs).
+The Demand department is directly responsible for achieving revenue pipeline targets, increasing awareness and interest in the open-source project, and nurturing the Fleet community through participation in video, sponsored events, and other [programs](https://fleetdm.com/handbook/company/communications#programs).
### Respond to a "Contact us" submission
1. Check the [_from-prospective-customers](https://fleetdm.slack.com/archives/C01HE9GQW6B) Slack channel for "Contact us" submissions.
diff --git a/handbook/digital-experience/README.md b/handbook/digital-experience/README.md
index 5f973ae726..f76e62ad2c 100644
--- a/handbook/digital-experience/README.md
+++ b/handbook/digital-experience/README.md
@@ -4,9 +4,10 @@ This page details processes specific to working [with](#contact-us) and [within]
## Team
| Role | Contributor(s)
|:--------------------------------|:----------------------------------------------------------------------|
-| Head of Digital Experience | [Sam Pfluger](https://www.linkedin.com/in/sampfluger88/) _([@sampfluger88](https://github.com/sampfluger88))_
-| Head of Design | [Mike Thomas](https://www.linkedin.com/in/mike-thomas-52277938) _([@mike-j-thomas](https://github.com/mike-j-thomas))_
-| Software Engineer | [Eric Shaw](https://www.linkedin.com/in/eric-shaw-1423831a9/) _([@eashaw](https://github.com/eashaw))_
+| Head of Digital Experience | See [🌐 Apprentice to the CEO](https://fleetdm.com/handbook/digital-experience#team)
+| Head of Design | [Mike Thomas](https://www.linkedin.com/in/mike-thomas-52277938) _([@mike-j-thomas](https://github.com/mike-j-thomas))_
+| Software Engineer | [Eric Shaw](https://www.linkedin.com/in/eric-shaw-1423831a9/) _([@eashaw](https://github.com/eashaw))_
+| Apprentice to the CEO | [Sam Pfluger](https://www.linkedin.com/in/sampfluger88/) _([@sampfluger88](https://github.com/sampfluger88))_
[Savannah Friend](https://www.linkedin.com/in/savannah-friend-2b1a53148/) _(@todo)_
## Contact us
@@ -15,11 +16,11 @@ This page details processes specific to working [with](#contact-us) and [within]
- Any Fleet team member can [view the kanban board](https://app.zenhub.com/workspaces/g-sales-64fbb46c65f9ff003a1530a8/board?sprints=none) for this department, including pending tasks and the status of new requests.
- Please **use issue comments and GitHub mentions** to communicate follow-ups or answer questions related to your request.
-> _**Note:** If a user story involves only changes to fleetdm.com, without changing the core product, then that user story is prioritized, drafted, implemented, and shipped by the [Digital Experience](https://fleetdm.com/handbook/digital-experience) department. Otherwise, if the story **also** involves changes to the core product **as well as** fleetdm.com, then that user story is prioritized, drafted, implemented, and shipped by [the other relevant product group](https://fleetdm.com/handbook/company/product-groups#current-product-groups), and not by `#g-digital-experience`._
## Responsibilities
The Digital Experience department is directly responsible for the framework, content design, and technology behind Fleet's remote work culture, including fleetdm.com, the handbook, issue templates, UI style guides, internal tooling, Zapier flows, Docusign templates, key spreadsheets, and project management processes.
+> _**Note:** If a user story involves only changes to fleetdm.com, without changing the core product, then that user story is prioritized, drafted, implemented, and shipped by the [Digital Experience](https://fleetdm.com/handbook/digital-experience) department. Otherwise, if the story **also** involves changes to the core product **as well as** fleetdm.com, then that user story is prioritized, drafted, implemented, and shipped by [the other relevant product group](https://fleetdm.com/handbook/company/product-groups#current-product-groups), and not by `#g-digital-experience`._
### QA a change to fleetdm.com
Each PR to the website is manually checked for quality and tested before before going live on fleetdm.com. To test any change to fleetdm.com
diff --git a/handbook/engineering/README.md b/handbook/engineering/README.md
index 2b72932d2b..f8262fae36 100644
--- a/handbook/engineering/README.md
+++ b/handbook/engineering/README.md
@@ -1,13 +1,15 @@
# Engineering
-This handbook page details processes specific to working [with](#team) and [within](#responsibilities) this department
+This handbook page details processes specific to working [with](#contact-us) and [within](#responsibilities) this department.
## Team
| Role | Contributor(s) |
|:--------------------------------|:-----------------------------------------------------------------------------------------------------------|
-| CTO | [Luke Heath](https://www.linkedin.com/in/lukeheath/) _([@lukeheath](https://github.com/lukeheath))_
-| Engineering Manager | _See ["Current product groups"](https://fleetdm.com/handbook/company/product-groups#current-product-groups)_
-| Quality Assurance | [Reed Haynes](https://www.linkedin.com/in/reed-haynes-633a69a3/) _([@xpkoala](https://github.com/xpkoala))_
-| Developer | _See ["Current product groups"](https://fleetdm.com/handbook/company/product-groups#current-product-groups)_
+| Chief Technology Officer (CTO) | [Luke Heath](https://www.linkedin.com/in/lukeheath/) _([@lukeheath](https://github.com/lukeheath))_
+| Client Platform Engineer & Community Advocate | [JD Strong](https://www.linkedin.com/in/jackdaniyelstrong/) _([@spokanemac](https://github.com/spokanemac/spokanemac))_
+| Engineering Manager (EM) | _See [🛩️ Product groups](https://fleetdm.com/handbook/company/product-groups#current-product-groups)_
+| Quality Assurance Engineer (QA) | _See [🛩️ Product groups](https://fleetdm.com/handbook/company/product-groups#current-product-groups)_
+| Software Engineer | _See [🛩️ Product groups](https://fleetdm.com/handbook/company/product-groups#current-product-groups)_
+
## Contact us
- To **make a request** of this department, [create an issue](https://fleetdm.com/handbook/company/product-groups#current-product-groups) and a team member will get back to you within one business day (If urgent, mention a [team member](#team) in the [#help-engineering](https://fleetdm.slack.com/archives/C019WG4GH0A) Slack channel.
@@ -48,7 +50,7 @@ If the bug is labeled `~unreleased bug`, branch off and put your PR into `main`.
If the bug is labeled `~released bug`, branch off the tag for the latest release of Fleet and put your PR into `main`. For example, `git checkout fleet-v4.48.2`, then `git checkout -b my-bug-fix-branch`. These issues are not closed until the next release of Fleet. This approach makes sure the bug fix is not built on top of unreleased feature code, which can cause merge conflicts during patch releases.
### Begin a merge freeze
-To ensure release quality, Fleet has a freeze period for testing beginning the Tuesday before the release at 9:00 AM Pacific. Effective at the start of the freeze period, new feature work will not be merged into `main`.
+To ensure release quality, Fleet has a freeze period for testing beginning the Tuesday before the release at 11:00 AM Pacific. Effective at the start of the freeze period, new feature work will not be merged into `main`.
Bugs are exempt from the release freeze period.
diff --git a/handbook/product-design/README.md b/handbook/product-design/README.md
index a560a0764e..a224150441 100644
--- a/handbook/product-design/README.md
+++ b/handbook/product-design/README.md
@@ -1,14 +1,11 @@
-# Product design
+# Product Design
This handbook page details processes specific to working [with](#contact-us) and [within](#responsibilities) this department.
## Team
-| Role | Contributor(s) |
-|:--------------------------------|:-----------------------------------------------------------------------------------------------------------|
+| Role | Contributor(s) |
+|:---------------------------------|:-----------------------------------------------------------------------------------------------------------|
| Head of Product Design | [Noah Talerman](https://www.linkedin.com/in/noah-talerman/) _([@noahtalerman](https://github.com/noahtalerman))_
-| Head of Design | [Mike Thomas](https://www.linkedin.com/in/mike-thomas-52277938) _([@mike-j-thomas](https://github.com/mike-j-thomas))_
-| Product Designer | [Rachael Shaw](https://www.linkedin.com/in/rachaelcshaw/) _([@rachaelshaw](https://github.com/rachaelshaw))_, [Marko Lisica](https://www.linkedin.com/in/markolisica/) _([@marko-lisica](https://github.com/marko-lisica))_
-| Developer | [Eric Shaw](https://www.linkedin.com/in/eric-shaw-1423831a9/) _([@eashaw](https://github.com/eashaw))_
-
+| Product Designer | _See [🛩️ Product groups](https://fleetdm.com/handbook/company/product-groups#current-product-groups)_
## Contact us
- To **make a request** of this department, [create an issue](https://github.com/fleetdm/confidential/issues/new?labels=%3Aproduct&title=Product%20design%20request%C2%BB______________________&template=custom-request.md) and a team member will get back to you within one business day (If urgent, mention a [team member](#team) in `#help-design`.
diff --git a/handbook/sales/README.md b/handbook/sales/README.md
index b1fe017ad4..6201676cc5 100644
--- a/handbook/sales/README.md
+++ b/handbook/sales/README.md
@@ -4,12 +4,11 @@ This handbook page details processes specific to working [with](#contact-us) and
## Team
| Role | Contributor(s) |
|:--------------------------------------|:------------------------------------------------------------------------------------------------------------------------|
-| Chief Revenue Officer (CRO) | [Alex Mitchell](https://www.linkedin.com/in/alexandercmitchell/) _([@alexmitchelliii](https://github.com/alexmitchelliii))_
-| 🏹 [Customer Success](https://www.fleetdm.com/handbook/customer-success#responsibilities) | [Customer Success team members](https://www.fleetdm.com/handbook/customer-success#team)
-| Director of Solutions Consulting | [Dave Herder](https://www.linkedin.com/in/daveherder/) _([@dherder](https://github.com/dherder))_
-| Solutions Consultant (SC) | [Will Mayhone](https://www.linkedin.com/in/william-mayhone-671977b6/) _([@willmayhone88](https://github.com/willmayhone88))_
-| Head of Public Sector | [Keith Barnes](https://www.linkedin.com/in/keith-barnes-8b666/) _([@KAB703](https://github.com/KAB703))_
-| Account Executive (AE) | [Tom Ostertag](https://www.linkedin.com/in/tom-ostertag-77212791/) _([@TomOstertag](https://github.com/TomOstertag))_, [Patricia Ambrus](https://www.linkedin.com/in/pambrus/) _([@ambrusps](https://github.com/ambrusps))_, [Anthony Snyder](https://www.linkedin.com/in/anthonysnyder8/) _([@AnthonySnyder8](https://github.com/AnthonySnyder8))_, [Paul Tardif](https://www.linkedin.com/in/paul-t-750833/) _([@phtardif1](https://github.com/phtardif1))_
+| Chief Revenue Officer (CRO) | [Alex Mitchell](https://www.linkedin.com/in/alexandercmitchell/) _([@alexmitchelliii](https://github.com/alexmitchelliii))_
+| Solutions Consulting (SC) | [Dave Herder](https://www.linkedin.com/in/daveherder/) _([@dherder](https://github.com/dherder))_
[Zach Wasserman](https://www.linkedin.com/in/zacharywasserman/) _([@zwass](https://github.com/zwass))_
[Will Mayhone](https://www.linkedin.com/in/william-mayhone-671977b6/) _([@willmayhone88](https://github.com/willmayhone88))_
+| Public Sector | [Keith Barnes](https://www.linkedin.com/in/keith-barnes-8b666/) _([@KAB703](https://github.com/KAB703))_
+| Channel Sales | [Tom Ostertag](https://www.linkedin.com/in/tom-ostertag-77212791/) _([@tomostertag](https://github.com/TomOstertag))_
+| Account Executive (AE) | [Patricia Ambrus](https://www.linkedin.com/in/pambrus/) _([@ambrusps](https://github.com/ambrusps))_
[Anthony Snyder](https://www.linkedin.com/in/anthonysnyder8/) _([@anthonysnyder8](https://github.com/AnthonySnyder8))_
[Paul Tardif](https://www.linkedin.com/in/paul-t-750833/) _([@phtardif1](https://github.com/phtardif1))_
## Contact us
diff --git a/infrastructure/dogfood/terraform/aws-tf-module/free.tf b/infrastructure/dogfood/terraform/aws-tf-module/free.tf
index 4efc3999b9..ec6aeaa454 100644
--- a/infrastructure/dogfood/terraform/aws-tf-module/free.tf
+++ b/infrastructure/dogfood/terraform/aws-tf-module/free.tf
@@ -25,6 +25,7 @@ module "free" {
}
rds_config = {
name = local.customer_free
+ engine_version = "8.0.mysql_aurora.3.05.2"
snapshot_identifier = "arn:aws:rds:us-east-2:611884880216:cluster-snapshot:a2023-03-06-pre-migration"
db_parameters = {
# 8mb up from 262144 (256k) default
diff --git a/infrastructure/dogfood/terraform/aws-tf-module/main.tf b/infrastructure/dogfood/terraform/aws-tf-module/main.tf
index 2b0056fc79..a3c3614abb 100644
--- a/infrastructure/dogfood/terraform/aws-tf-module/main.tf
+++ b/infrastructure/dogfood/terraform/aws-tf-module/main.tf
@@ -70,6 +70,7 @@ module "main" {
}
rds_config = {
name = local.customer
+ engine_version = "8.0.mysql_aurora.3.05.2"
snapshot_identifier = "arn:aws:rds:us-east-2:611884880216:cluster-snapshot:a2023-03-06-pre-migration"
db_parameters = {
# 8mb up from 262144 (256k) default
diff --git a/infrastructure/dogfood/terraform/aws/rds.tf b/infrastructure/dogfood/terraform/aws/rds.tf
index 10e0fb512a..bd37ea9cf9 100644
--- a/infrastructure/dogfood/terraform/aws/rds.tf
+++ b/infrastructure/dogfood/terraform/aws/rds.tf
@@ -67,7 +67,7 @@ module "aurora_mysql" {
name = "${local.name}-mysql-iam"
engine = "aurora-mysql"
- engine_version = "8.0.mysql_aurora.3.02.0"
+ engine_version = "8.0.mysql_aurora.3.05.2"
instance_type = var.db_instance_type_writer
instance_type_replica = var.db_instance_type_reader
diff --git a/infrastructure/dogfood/terraform/aws/variables.tf b/infrastructure/dogfood/terraform/aws/variables.tf
index 592383ad2d..83f3879dac 100644
--- a/infrastructure/dogfood/terraform/aws/variables.tf
+++ b/infrastructure/dogfood/terraform/aws/variables.tf
@@ -56,7 +56,7 @@ variable "database_name" {
variable "fleet_image" {
description = "the name of the container image to run"
- default = "fleetdm/fleet:v4.48.3"
+ default = "fleetdm/fleet:v4.49.1"
}
variable "software_inventory" {
diff --git a/infrastructure/dogfood/terraform/gcp/variables.tf b/infrastructure/dogfood/terraform/gcp/variables.tf
index a1b08bab80..7de4a7f02f 100644
--- a/infrastructure/dogfood/terraform/gcp/variables.tf
+++ b/infrastructure/dogfood/terraform/gcp/variables.tf
@@ -68,5 +68,5 @@ variable "redis_mem" {
}
variable "image" {
- default = "fleet:v4.48.3"
+ default = "fleet:v4.49.1"
}
diff --git a/it-and-security/lib/collect-windows-defender.queries.yml b/it-and-security/lib/collect-windows-defender.queries.yml
new file mode 100644
index 0000000000..739dfcd999
--- /dev/null
+++ b/it-and-security/lib/collect-windows-defender.queries.yml
@@ -0,0 +1,10 @@
+- name: Collect Windows Defender
+ automations_enabled: false
+ description: Collects the pid, process name, user, path and command line for Windows Defender installed on hosts.
+ discard_data: false
+ interval: 3600
+ logging: snapshot
+ min_osquery_version: ""
+ observer_can_run: true
+ platform: "windows"
+ query: SELECT processes.pid, processes.name, users.username, processes.path, processes.cmdline FROM processes LEFT JOIN users ON processes.uid = users.uid WHERE processes.path != '' AND name LIKE 'MpCmdRun.exe';
\ No newline at end of file
diff --git a/it-and-security/lib/macos-device-health.policies.yml b/it-and-security/lib/macos-device-health.policies.yml
index 427ac7a149..f06480bd50 100644
--- a/it-and-security/lib/macos-device-health.policies.yml
+++ b/it-and-security/lib/macos-device-health.policies.yml
@@ -64,12 +64,3 @@
description: Looks for PDF files with file names typically used by 1Password for emergency recovery kits. To protect the performance of your devices, the search is one level deep and limited to the Desktop, Documents, Downloads, and Shared folders.
resolution: Delete 1Password emergency kits from your computer, and empty the trash. 1Password emergency kits should only be printed and stored in a physically secure location.
platform: darwin
-- name: macOS - Check if latest version
- query: |
- SELECT 1 FROM os_version
- WHERE (major > 14 OR (major = 14 AND minor > 4) OR (major = 14 AND minor = 4 AND patch >= 1)) --Sonoma
- critical: false
- description: This policy check if macOS version is most recent version available.
- resolution: From the Apple menu, select System Settings. Navigate to General > Software Update.
- platform: darwin
- calendar_events_enabled: true
diff --git a/it-and-security/lib/windows-device-health.policies.yml b/it-and-security/lib/windows-device-health.policies.yml
index 5a15b90cc5..40e276caae 100644
--- a/it-and-security/lib/windows-device-health.policies.yml
+++ b/it-and-security/lib/windows-device-health.policies.yml
@@ -22,4 +22,10 @@
description: This policy checks if the end user is required to enter a password, with at least 10 characters, to unlock the host.
resolution: "As an IT admin, deploy a Windows profile with the DevicePasswordEnabled and MinDevicePasswordLength option documented here: https://learn.microsoft.com/en-us/windows/client-management/mdm/policy-csp-devicelock"
platform: windows
+- name: Windows - Antivirus healthy
+ query: SELECT 1 from windows_security_center wsc CROSS JOIN windows_security_products wsp WHERE antivirus = 'Good' AND type = 'Antivirus' AND signatures_up_to_date=1;
+ critical: false
+ description: Checks the status of antivirus and signature updates from the Windows Security Center.
+ resolution: "Ensure Windows Defender or your third-party antivirus is running, up to date, and visible in the Windows Security Center."
+ platform: windows
diff --git a/it-and-security/mdm-commands/apple/send-fleetd.xml b/it-and-security/mdm-commands/apple/send-fleetd.xml
new file mode 100644
index 0000000000..0a10ff6d99
--- /dev/null
+++ b/it-and-security/mdm-commands/apple/send-fleetd.xml
@@ -0,0 +1,15 @@
+
+
+
+ Command
+
+ ManifestURL
+ https://download.fleetdm.com/fleetd-base-manifest.plist
+ RequestType
+ InstallEnterpriseApplication
+
+
+ CommandUUID
+ adc1bc23-abec-4499-b57f-c8755c7ffe3c
+
+
diff --git a/it-and-security/teams/workstations-canary.yml b/it-and-security/teams/workstations-canary.yml
index 4f6c62a93a..5d9334c09c 100644
--- a/it-and-security/teams/workstations-canary.yml
+++ b/it-and-security/teams/workstations-canary.yml
@@ -109,6 +109,15 @@ policies:
- path: ../lib/macos-device-health.policies.yml
- path: ../lib/windows-device-health.policies.yml
- path: ../lib/linux-device-health.policies.yml
+ - name: macOS - Check if latest version
+ query: |
+ SELECT 1 FROM os_version
+ WHERE (major > 14 OR (major = 14 AND minor > 4) OR (major = 14 AND minor = 4 AND patch >= 2)) --Sonoma
+ critical: false
+ description: This policy check if macOS version is most recent version available.
+ resolution: From the Apple menu, select System Settings. Navigate to General > Software Update.
+ platform: darwin
+ calendar_events_enabled: true
queries:
- path: ../lib/collect-failed-login-attempts.queries.yml
- path: ../lib/collect-fleetd-information.yml
diff --git a/it-and-security/teams/workstations.yml b/it-and-security/teams/workstations.yml
index 95ca009dc3..491eab9211 100644
--- a/it-and-security/teams/workstations.yml
+++ b/it-and-security/teams/workstations.yml
@@ -8,6 +8,10 @@ team_settings:
host_expiry_window: 0
secrets:
- secret: $DOGFOOD_WORKSTATIONS_ENROLL_SECRET
+ integrations:
+ google_calendar:
+ enable_calendar_events: true
+ webhook_url: $DOGFOOD_WORKSTATIONS_CANARY_CALENDAR_WEBHOOK_URL
agent_options:
path: ../lib/agent-options.yml
controls:
@@ -57,6 +61,15 @@ policies:
- path: ../lib/macos-device-health.policies.yml
- path: ../lib/windows-device-health.policies.yml
- path: ../lib/linux-device-health.policies.yml
+ - name: macOS - Check if latest version
+ query: |
+ SELECT 1 FROM os_version
+ WHERE (major > 14 OR (major = 14 AND minor > 4) OR (major = 14 AND minor = 4 AND patch >= 1)) --Sonoma
+ critical: false
+ description: This policy check if macOS version is most recent version available.
+ resolution: From the Apple menu, select System Settings. Navigate to General > Software Update.
+ platform: darwin
+ calendar_events_enabled: true
queries:
- path: ../lib/collect-failed-login-attempts.queries.yml
- path: ../lib/collect-usb-devices.queries.yml
diff --git a/orbit/TUF.md b/orbit/TUF.md
index ff9a89953f..64d8bb4a1b 100644
--- a/orbit/TUF.md
+++ b/orbit/TUF.md
@@ -7,8 +7,8 @@ Following are the currently deployed versions of fleetd components on the `stabl
| Component\OS | macOS | Linux | Windows |
|--------------|--------------|--------|---------|
-| orbit | 1.23.0 | 1.23.0 | 1.23.0 |
-| desktop | 1.23.0 | 1.23.0 | 1.23.0 |
+| orbit | 1.24.0 | 1.24.0 | 1.24.0 |
+| desktop | 1.24.0 | 1.24.0 | 1.24.0 |
| osqueryd | 5.12.1 | 5.12.1 | 5.12.1 |
| nudge | 1.1.10.81462 | - | - |
| swiftDialog | 2.1.0 | - | - |
diff --git a/orbit/changes/17187-sign-windows b/orbit/changes/17187-sign-windows
new file mode 100644
index 0000000000..b074822855
--- /dev/null
+++ b/orbit/changes/17187-sign-windows
@@ -0,0 +1 @@
+Windows orbit.exe and fleet-desktop.exe are now signed.
diff --git a/package.json b/package.json
index f9e4ec1145..29ac16831a 100644
--- a/package.json
+++ b/package.json
@@ -58,6 +58,7 @@
"sqlite-parser": "1.0.1",
"use-debounce": "9.0.4",
"uuid": "8.3.2",
+ "validator": "13.11.0",
"when": "3.7.8"
},
"devDependencies": {
@@ -91,14 +92,15 @@
"@storybook/react": "7.5.2",
"@storybook/react-webpack5": "7.5.2",
"@storybook/test-runner": "0.13.0",
- "@testing-library/jest-dom": "5.16.2",
- "@testing-library/react": "12.1.4",
- "@testing-library/user-event": "14.4.3",
+ "@testing-library/jest-dom": "6.4.2",
+ "@testing-library/react": "15.0.2",
+ "@testing-library/user-event": "14.5.2",
"@tsconfig/recommended": "1.0.1",
"@types/chrome": "0.0.237",
"@types/classnames": "0.0.32",
"@types/expect": "1.20.3",
"@types/file-saver": "2.0.5",
+ "@types/jest": "29.5.12",
"@types/js-md5": "0.4.3",
"@types/js-yaml": "4.0.5",
"@types/lodash": "4.14.179",
@@ -115,6 +117,7 @@
"@types/react-tooltip": "4.2.4",
"@types/sockjs-client": "1.5.1",
"@types/uuid": "8.3.4",
+ "@types/validator": "13.11.9",
"@typescript-eslint/eslint-plugin": "5.58.0",
"@typescript-eslint/parser": "5.58.0",
"autoprefixer": "10.4.19",
diff --git a/pkg/mdm/mdmtest/apple.go b/pkg/mdm/mdmtest/apple.go
index a2c2f46e33..d4463575d4 100644
--- a/pkg/mdm/mdmtest/apple.go
+++ b/pkg/mdm/mdmtest/apple.go
@@ -135,7 +135,7 @@ func NewTestMDMClientAppleDEP(serverURL string, depURLToken string, opts ...Test
return &c
}
-// NewTestMDMClientDEP will create a simulated device that will not fetch the enrollment
+// NewTestMDMClientAppleDirect will create a simulated device that will not fetch the enrollment
// profile from Fleet. The enrollment information is to be provided in the enrollInfo.
func NewTestMDMClientAppleDirect(enrollInfo AppleEnrollInfo, opts ...TestMDMAppleClientOption) *TestAppleMDMClient {
c := TestAppleMDMClient{
@@ -151,6 +151,14 @@ func NewTestMDMClientAppleDirect(enrollInfo AppleEnrollInfo, opts ...TestMDMAppl
return &c
}
+func (c *TestAppleMDMClient) SetDesktopToken(tok string) {
+ c.desktopURLToken = tok
+}
+
+func (c *TestAppleMDMClient) SetDEPToken(tok string) {
+ c.depURLToken = tok
+}
+
// Enroll runs the MDM enroll protocol on the simulated device.
func (c *TestAppleMDMClient) Enroll() error {
switch {
diff --git a/pkg/mdm/mdmtest/windows.go b/pkg/mdm/mdmtest/windows.go
index 91bd5d2319..fc504d1bca 100644
--- a/pkg/mdm/mdmtest/windows.go
+++ b/pkg/mdm/mdmtest/windows.go
@@ -22,8 +22,8 @@ import (
type TestWindowsMDMClient struct {
// DeviceID identifies a MDM enrollment, sent and managed by the device.
DeviceID string
- // hardwareID identifies a device.
- hardwareID string
+ // HardwareID identifies a device.
+ HardwareID string
// fleetServerURL is the URL of the Fleet server, used to ping the MDM endpoints.
fleetServerURL string
// debug enables debug logging of request/responses.
@@ -31,8 +31,8 @@ type TestWindowsMDMClient struct {
// enrollmentType is used to simulate different Windows enrollment
// types (programatic, automatic.)
enrollmentType fleet.WindowsMDMEnrollmentType
- // tokenIdentifier is used for authentication during the programmatic enrollment.
- tokenIdentifier string
+ // TokenIdentifier is used for authentication during the programmatic enrollment.
+ TokenIdentifier string
// lastManagementResp tracks the last response we received from the server.
lastManagementResp *fleet.SyncML
// queuedCommandResponses tracks the commands that will be sent next
@@ -57,8 +57,8 @@ func NewTestMDMClientWindowsProgramatic(serverURL string, orbitNodeKey string, o
fleetServerURL: serverURL,
DeviceID: uuid.NewString(),
enrollmentType: fleet.WindowsMDMProgrammaticEnrollmentType,
- tokenIdentifier: orbitNodeKey,
- hardwareID: uuid.NewString(),
+ TokenIdentifier: orbitNodeKey,
+ HardwareID: uuid.NewString(),
}
for _, fn := range opts {
fn(&c)
@@ -71,8 +71,8 @@ func NewTestMDMClientWindowsAutomatic(serverURL string, email string, opts ...Te
fleetServerURL: serverURL,
DeviceID: uuid.NewString(),
enrollmentType: fleet.WindowsMDMAutomaticEnrollmentType,
- tokenIdentifier: email,
- hardwareID: uuid.NewString(),
+ TokenIdentifier: email,
+ HardwareID: uuid.NewString(),
}
for _, fn := range opts {
fn(&c)
@@ -319,7 +319,7 @@ YioVozr1IWYySwWVzMf/SUwKZkKJCAJmSVcixE+4kxPkyPGyauIrN3wWC0zb+mjF
false
- ` + c.hardwareID + `
+ ` + c.HardwareID + `
en-US
@@ -487,7 +487,7 @@ func (c *TestWindowsMDMClient) getToken() (binarySecToken string, tokenValueType
switch c.enrollmentType {
case fleet.WindowsMDMAutomaticEnrollmentType:
claims := &jwt.MapClaims{
- "upn": c.tokenIdentifier,
+ "upn": c.TokenIdentifier,
"tid": "tenant_id",
"unique_name": "foo_bar",
"scp": "mdm_delegation",
@@ -504,7 +504,7 @@ func (c *TestWindowsMDMClient) getToken() (binarySecToken string, tokenValueType
case fleet.WindowsMDMProgrammaticEnrollmentType:
var err error
tokenValueType = syncml.BinarySecurityDeviceEnroll
- binarySecToken, err = fleet.GetEncodedBinarySecurityToken(c.enrollmentType, c.tokenIdentifier)
+ binarySecToken, err = fleet.GetEncodedBinarySecurityToken(c.enrollmentType, c.TokenIdentifier)
if err != nil {
return "", "", fmt.Errorf("generating encoded security token: %w", err)
}
diff --git a/pkg/spec/gitops_test.go b/pkg/spec/gitops_test.go
index 7a94695c63..98a0a36a0c 100644
--- a/pkg/spec/gitops_test.go
+++ b/pkg/spec/gitops_test.go
@@ -2,13 +2,14 @@ package spec
import (
"fmt"
- "github.com/fleetdm/fleet/v4/server/fleet"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
"os"
"path/filepath"
"slices"
"testing"
+
+ "github.com/fleetdm/fleet/v4/server/fleet"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
)
var topLevelOptions = map[string]string{
@@ -99,6 +100,7 @@ func TestValidGitOpsYaml(t *testing.T) {
assert.Contains(t, gitops.OrgSettings, "webhook_settings")
assert.Contains(t, gitops.OrgSettings, "fleet_desktop")
assert.Contains(t, gitops.OrgSettings, "host_expiry_settings")
+ assert.Contains(t, gitops.OrgSettings, "activity_expiry_settings")
assert.Contains(t, gitops.OrgSettings, "features")
assert.Contains(t, gitops.OrgSettings, "vulnerability_settings")
assert.Contains(t, gitops.OrgSettings, "secrets")
@@ -107,6 +109,14 @@ func TestValidGitOpsYaml(t *testing.T) {
require.Len(t, secrets.([]*fleet.EnrollSecret), 2)
assert.Equal(t, "SampleSecret123", secrets.([]*fleet.EnrollSecret)[0].Secret)
assert.Equal(t, "ABC", secrets.([]*fleet.EnrollSecret)[1].Secret)
+ activityExpirySettings, ok := gitops.OrgSettings["activity_expiry_settings"].(map[string]interface{})
+ require.True(t, ok)
+ activityExpiryEnabled, ok := activityExpirySettings["activity_expiry_enabled"].(bool)
+ require.True(t, ok)
+ require.True(t, activityExpiryEnabled)
+ activityExpiryWindow, ok := activityExpirySettings["activity_expiry_window"].(float64)
+ require.True(t, ok)
+ require.Equal(t, 30, int(activityExpiryWindow))
}
// Check controls
@@ -144,7 +154,6 @@ func TestValidGitOpsYaml(t *testing.T) {
assert.Equal(t, "No root logins (macOS, Linux)", gitops.Policies[2].Name)
assert.Equal(t, "🔥 Failing policy", gitops.Policies[3].Name)
assert.Equal(t, "😊😊 Failing policy", gitops.Policies[4].Name)
-
},
)
}
@@ -239,7 +248,6 @@ func TestMixingGlobalAndTeamConfig(t *testing.T) {
config += "team_settings:\n secrets: []\n"
_, err = GitOpsFromBytes([]byte(config), "")
assert.ErrorContains(t, err, "'org_settings' cannot be used with 'name' or 'team_settings'")
-
}
func TestInvalidGitOpsYaml(t *testing.T) {
diff --git a/pkg/spec/testdata/global_config_no_paths.yml b/pkg/spec/testdata/global_config_no_paths.yml
index 4c4ee3eb7d..cdc6e78923 100644
--- a/pkg/spec/testdata/global_config_no_paths.yml
+++ b/pkg/spec/testdata/global_config_no_paths.yml
@@ -171,6 +171,9 @@ org_settings:
transparency_url: https://fleetdm.com/transparency
host_expiry_settings: # Applies to all teams
host_expiry_enabled: false
+ activity_expiry_settings:
+ activity_expiry_enabled: true
+ activity_expiry_window: 30
features: # Features added to all teams
enable_host_users: true
enable_software_inventory: true
diff --git a/pkg/spec/testdata/org-settings.yml b/pkg/spec/testdata/org-settings.yml
index 98038b2077..17855a1b8b 100644
--- a/pkg/spec/testdata/org-settings.yml
+++ b/pkg/spec/testdata/org-settings.yml
@@ -74,6 +74,9 @@ fleet_desktop: # Applies to Fleet Premium only
transparency_url: https://fleetdm.com/transparency
host_expiry_settings: # Applies to all teams
host_expiry_enabled: false
+activity_expiry_settings:
+ activity_expiry_enabled: true
+ activity_expiry_window: 30
features: # Features added to all teams
enable_host_users: true
enable_software_inventory: true
diff --git a/schema/tables/apfs_physical_stores.yml b/schema/tables/apfs_physical_stores.yml
index f9c8f1992e..7492e8e7b0 100644
--- a/schema/tables/apfs_physical_stores.yml
+++ b/schema/tables/apfs_physical_stores.yml
@@ -41,5 +41,5 @@ columns:
type: bigint
required: false
description: The size of the physical store in byptes
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/apfs_volumes.yml b/schema/tables/apfs_volumes.yml
index bfb3f1d2a8..3a94e6c535 100644
--- a/schema/tables/apfs_volumes.yml
+++ b/schema/tables/apfs_volumes.yml
@@ -75,5 +75,5 @@ columns:
type: integer
required: false
description: Whether the volume is unreadable because it does not have a key entered
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/authdb.yml b/schema/tables/authdb.yml
index f016ed3084..1785b333d3 100644
--- a/schema/tables/authdb.yml
+++ b/schema/tables/authdb.yml
@@ -13,5 +13,5 @@ columns:
required: false
description: >-
The JSON output parsed from the plist output of the `authorizationdb read ` command.
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/cis_audit.yml b/schema/tables/cis_audit.yml
index 79055e0bda..343cdad5f9 100644
--- a/schema/tables/cis_audit.yml
+++ b/schema/tables/cis_audit.yml
@@ -11,5 +11,5 @@ columns:
type: text
required: false
description: Contains the value for the queried CIS item.
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/corestorage_logical_volume_families.yml b/schema/tables/corestorage_logical_volume_families.yml
index dafa9e333e..09cc50ecaf 100644
--- a/schema/tables/corestorage_logical_volume_families.yml
+++ b/schema/tables/corestorage_logical_volume_families.yml
@@ -76,5 +76,5 @@ columns:
type: integer
required: false
description: Whether a password is currently required to unlock the volume
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
\ No newline at end of file
diff --git a/schema/tables/corestorage_logical_volumes.yml b/schema/tables/corestorage_logical_volumes.yml
index 48558a5cbd..eb3c7d5f86 100644
--- a/schema/tables/corestorage_logical_volumes.yml
+++ b/schema/tables/corestorage_logical_volumes.yml
@@ -129,7 +129,7 @@ columns:
type: text
required: false
description: Name of the filesystem in the logical volume
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/csrutil_info.yml b/schema/tables/csrutil_info.yml
index cbc21425ca..6a903e3209 100644
--- a/schema/tables/csrutil_info.yml
+++ b/schema/tables/csrutil_info.yml
@@ -11,5 +11,5 @@ columns:
During system installation, a SHA-256 cryptographic hash is calculated for all immutable system files and stored in a Merkle tree which itself is hashed as the Seal. Both are stored in the metadata of the snapshot created of the System volume.
The seal is verified by the boot loader at startup. macOS will not boot if system files have been tampered with. If validation fails, the user will be instructed to reinstall the operating system.
During read operations for files located in the Sealed System Volume, a hash is calculated and compared to the value stored in the Merkle tree.
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/dscl.yml b/schema/tables/dscl.yml
index a17ae334bd..6698abba00 100644
--- a/schema/tables/dscl.yml
+++ b/schema/tables/dscl.yml
@@ -19,5 +19,5 @@ columns:
type: text
required: false
description: The value of the read path and key. The value is the empty string if the key doesn't exist.
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/falconctl_options.yml b/schema/tables/falconctl_options.yml
index 9e839c82eb..6be656af4b 100644
--- a/schema/tables/falconctl_options.yml
+++ b/schema/tables/falconctl_options.yml
@@ -6,6 +6,6 @@ platforms:
- linux
columns:
- name: options
- description: "The falconctol options to run. Supported values are listed here: `--aid`, `--apd`,`--aph`, `--app`, `--cid`, `--feature`, `--metadata-query`, `--rfm-reason`,`--rfm-state`, `--tags`, `--version`"
+ description: "The falconctl options to run. Supported values are listed here: `--aid`, `--apd`,`--aph`, `--app`, `--cid`, `--feature`, `--metadata-query`, `--rfm-reason`,`--rfm-state`, `--tags`, `--version`"
type: text
- required: true
\ No newline at end of file
+ required: true
diff --git a/schema/tables/file_lines.yml b/schema/tables/file_lines.yml
index 71c3fe0760..1f8b2413c4 100644
--- a/schema/tables/file_lines.yml
+++ b/schema/tables/file_lines.yml
@@ -1,5 +1,5 @@
name: file_lines
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
description: Allows reading an arbitrary file.
platforms:
- darwin
diff --git a/schema/tables/filevault_prk.yml b/schema/tables/filevault_prk.yml
index bdab68851f..8797ca1276 100644
--- a/schema/tables/filevault_prk.yml
+++ b/schema/tables/filevault_prk.yml
@@ -7,5 +7,5 @@ columns:
type: text
required: false
description: The base64-encoded contents of the encrypted FileVault personal recovery key stored at `/var/db/FileVaultPRK.dat` (see also https://developer.apple.com/documentation/devicemanagement/fderecoverykeyescrow)
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/filevault_users.yml b/schema/tables/filevault_users.yml
index d085092284..755d0f7532 100644
--- a/schema/tables/filevault_users.yml
+++ b/schema/tables/filevault_users.yml
@@ -1,5 +1,5 @@
name: filevault_users
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
description: Information on the users able to unlock the current boot volume if protected with FileVault.
platforms:
- darwin
diff --git a/schema/tables/find_cmd.yml b/schema/tables/find_cmd.yml
index e3d62df34e..0393688453 100644
--- a/schema/tables/find_cmd.yml
+++ b/schema/tables/find_cmd.yml
@@ -23,7 +23,5 @@ columns:
required: false
description: >-
Contains the found paths.
-notes: >-
- This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet.
- Fleetd installers can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/firmware_eficheck_integrity_check.yml b/schema/tables/firmware_eficheck_integrity_check.yml
index 49093b7802..493947378b 100644
--- a/schema/tables/firmware_eficheck_integrity_check.yml
+++ b/schema/tables/firmware_eficheck_integrity_check.yml
@@ -15,5 +15,5 @@ columns:
description: >-
Output of the `/usr/libexec/firmwarecheckers/eficheck/eficheck --integrity-check` command.
This value is only valid when chip is "intel-t1".
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/google_chrome_profiles.yml b/schema/tables/google_chrome_profiles.yml
index 1775ad9eb5..7d21c36e37 100644
--- a/schema/tables/google_chrome_profiles.yml
+++ b/schema/tables/google_chrome_profiles.yml
@@ -1,5 +1,5 @@
name: google_chrome_profiles
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
description: Profiles configured in Google Chrome.
platforms:
- darwin
diff --git a/schema/tables/icloud_private_relay.yml b/schema/tables/icloud_private_relay.yml
index e6f3f86d0b..cd80553e10 100644
--- a/schema/tables/icloud_private_relay.yml
+++ b/schema/tables/icloud_private_relay.yml
@@ -7,5 +7,5 @@ columns:
type: integer
required: false
description: whether iCloud Private Relay is on or off. 1 is on. 0 is off.
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/macadmins_unified_log.yml b/schema/tables/macadmins_unified_log.yml
index bd8c6a85af..b77a52c0f1 100644
--- a/schema/tables/macadmins_unified_log.yml
+++ b/schema/tables/macadmins_unified_log.yml
@@ -1,5 +1,5 @@
name: macadmins_unified_log
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
description: Allows querying macOS [unified logs](https://developer.apple.com/documentation/os/logging).
platforms:
- darwin
diff --git a/schema/tables/macos_profiles.yml b/schema/tables/macos_profiles.yml
index 782b63f387..18836ed229 100644
--- a/schema/tables/macos_profiles.yml
+++ b/schema/tables/macos_profiles.yml
@@ -1,5 +1,5 @@
name: macos_profiles
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
description: High level information on installed profiles enrollment.
platforms:
- darwin
diff --git a/schema/tables/macos_rsr.yml b/schema/tables/macos_rsr.yml
index 3bbe177f62..3338b6749d 100644
--- a/schema/tables/macos_rsr.yml
+++ b/schema/tables/macos_rsr.yml
@@ -1,5 +1,5 @@
name: macos_rsr
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
description: Returns information about installed Rapid Security Responses (RSRs).
platforms:
- darwin
diff --git a/schema/tables/mdm.yml b/schema/tables/mdm.yml
index abf9c15391..a63e09b13c 100644
--- a/schema/tables/mdm.yml
+++ b/schema/tables/mdm.yml
@@ -1,5 +1,10 @@
name: mdm
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer). Code based on work by [Kolide](https://github.com/kolide/launcher).
Due to changes in macOS 12.3, the output of `profiles show -type enrollment` can only be generated once a day. If you are running this command with another tool, you should set the `PROFILES_SHOW_ENROLLMENT_CACHE_PATH` environment variable to the path you are caching this. The cache file should be `json` with the keys `dep_capable` and `rate_limited present`, both booleans representing whether the device is capable of DEP enrollment and whether the response from `profiles show -type enrollment` is being rate limited or not.
+notes: >-
+ - This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
+
+ - Code based on work by [Kolide](https://github.com/kolide/launcher).
+
+ - Due to changes in macOS 12.3, the output of `profiles show -type enrollment` can only be generated once a day. If you are running this command with another tool, you should set the `PROFILES_SHOW_ENROLLMENT_CACHE_PATH` environment variable to the path you are caching this. The cache file should be `json` with the keys `dep_capable` and `rate_limited present`, both booleans representing whether the device is capable of DEP enrollment and whether the response from `profiles show -type enrollment` is being rate limited or not.
description: Information on the device's MDM enrollment.
platforms:
- darwin
diff --git a/schema/tables/mdm_bridge.yml b/schema/tables/mdm_bridge.yml
index 3656d7fba9..a247ae37e9 100644
--- a/schema/tables/mdm_bridge.yml
+++ b/schema/tables/mdm_bridge.yml
@@ -23,5 +23,5 @@ columns:
type: text
required: false
description: The full raw output of the MDM command execution.
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/munki_info.yml b/schema/tables/munki_info.yml
index 7c95b8d28e..b837daa4d7 100644
--- a/schema/tables/munki_info.yml
+++ b/schema/tables/munki_info.yml
@@ -1,5 +1,8 @@
name: munki_info
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
Code based on work by [macadmins/osquery-extension](https://github.com/macadmins/osquery-extension) and [Kolide](https://github.com/kolide/launcher).
+notes: >-
+ - This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
+
+ - Code based on work by [macadmins/osquery-extension](https://github.com/macadmins/osquery-extension) and [Kolide](https://github.com/kolide/launcher).
description: Information from the last [Munki](https://github.com/munki/munki) run.
platforms:
- darwin
diff --git a/schema/tables/munki_installs.yml b/schema/tables/munki_installs.yml
index 3f05dfc3fd..dfa49b6844 100644
--- a/schema/tables/munki_installs.yml
+++ b/schema/tables/munki_installs.yml
@@ -1,5 +1,8 @@
name: munki_installs
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
Code based on work by [macadmins/osquery-extension](https://github.com/macadmins/osquery-extension) and [Kolide](https://github.com/kolide/launcher).
+notes: >-
+ - This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
+
+ - Code based on work by [macadmins/osquery-extension](https://github.com/macadmins/osquery-extension) and [Kolide](https://github.com/kolide/launcher).
description: Software packages and other items [Munki](https://github.com/munki/munki) is managing.
platforms:
- darwin
diff --git a/schema/tables/nvram_info.yml b/schema/tables/nvram_info.yml
index 7a37e987c7..321788bee4 100644
--- a/schema/tables/nvram_info.yml
+++ b/schema/tables/nvram_info.yml
@@ -9,5 +9,5 @@ columns:
description: >-
Apple Mobile File Integrity (AMFI) was first released in macOS 10.12. The daemon and service block attempts to run unsigned code. AMFI uses lanchd, code signatures, certificates, entitlements, and provisioning profiles to create a filtered entitlement dictionary for an app. AMFI is the macOS kernel module that enforces code-signing and library validation.
Note: AMFI cannot be disabled with SIP enabled, but a change attempt can be made that will appear successful, and report incorrectly as successful. If the AMFI audit fails, and the SIP audit passes, this is still an issue the admin should research.
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/orbit_info.yml b/schema/tables/orbit_info.yml
index 1dfa3a9ff0..cc0d923682 100644
--- a/schema/tables/orbit_info.yml
+++ b/schema/tables/orbit_info.yml
@@ -45,5 +45,5 @@ columns:
type: integer
required: false
description: 1 if running scripts is enabled, 0 if disabled.
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/parse_ini.yml b/schema/tables/parse_ini.yml
index 3d33596f39..e5f2719af7 100644
--- a/schema/tables/parse_ini.yml
+++ b/schema/tables/parse_ini.yml
@@ -1,5 +1,5 @@
name: parse_ini
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
description: Parse a file as INI configuration.
platforms:
- darwin
diff --git a/schema/tables/parse_json.yml b/schema/tables/parse_json.yml
index 7a0e9a339a..fd1e62c526 100644
--- a/schema/tables/parse_json.yml
+++ b/schema/tables/parse_json.yml
@@ -1,5 +1,5 @@
name: parse_json
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
description: Parses an entire file as JSON. See `parse_jsonl` where multiple JSON documents are supported.
platforms:
- darwin
diff --git a/schema/tables/parse_jsonl.yml b/schema/tables/parse_jsonl.yml
index 7aae0eef65..c2664437f7 100644
--- a/schema/tables/parse_jsonl.yml
+++ b/schema/tables/parse_jsonl.yml
@@ -1,5 +1,5 @@
name: parse_jsonl
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
description: Parses each line of a file as a separate JSON document. See `parse_json` to treat an entire file as a single JSON document.
platforms:
- darwin
diff --git a/schema/tables/parse_xml.yml b/schema/tables/parse_xml.yml
index 21b3fe2a40..6f8ea31658 100644
--- a/schema/tables/parse_xml.yml
+++ b/schema/tables/parse_xml.yml
@@ -1,5 +1,5 @@
name: parse_xml
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
description: Parses a file as an XML document.
platforms:
- darwin
diff --git a/schema/tables/pmset.yml b/schema/tables/pmset.yml
index 2e975c495f..e65779f26c 100644
--- a/schema/tables/pmset.yml
+++ b/schema/tables/pmset.yml
@@ -11,7 +11,5 @@ columns:
type: text
required: false
description: Result of the command in JSON format.
-notes: >-
- This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet.
- Fleetd installers can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/puppet_info.yml b/schema/tables/puppet_info.yml
index 4e704fc4f4..81a22390b8 100644
--- a/schema/tables/puppet_info.yml
+++ b/schema/tables/puppet_info.yml
@@ -1,5 +1,5 @@
name: puppet_info
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
description: Information on the last [Puppet](https://puppet.com/) run. This table uses data from the `last_run_report` that Puppet creates.
platforms:
- darwin
diff --git a/schema/tables/puppet_logs.yml b/schema/tables/puppet_logs.yml
index 9b84f426e7..2600f21539 100644
--- a/schema/tables/puppet_logs.yml
+++ b/schema/tables/puppet_logs.yml
@@ -1,5 +1,5 @@
name: puppet_logs
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
description: Outputs [Puppet](https://puppet.com/) logs from the last run.
platforms:
- darwin
diff --git a/schema/tables/puppet_state.yml b/schema/tables/puppet_state.yml
index ad8db96d8d..55bf80ff34 100644
--- a/schema/tables/puppet_state.yml
+++ b/schema/tables/puppet_state.yml
@@ -1,5 +1,5 @@
name: puppet_state
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
description: State of every resource [Puppet](https://puppet.com/) is managing. This table uses data from the `last_run_report` that Puppet creates.
platforms:
- darwin
diff --git a/schema/tables/pwd_policy.yml b/schema/tables/pwd_policy.yml
index f6c0fd920b..9f9dac97fc 100644
--- a/schema/tables/pwd_policy.yml
+++ b/schema/tables/pwd_policy.yml
@@ -1,7 +1,7 @@
name: pwd_policy
platforms:
- darwin
-description: Password Policiy (e.g max failed password attempts).
+description: Password Policy (e.g., max failed password attempts).
columns:
- name: max_failed_attempts
type: integer
@@ -28,7 +28,5 @@ columns:
required: false
description: >-
This parameter indicates the minimum number of mixed characters in a password.
-notes: >-
- This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet.
- Fleetd installers can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/sntp_request.yml b/schema/tables/sntp_request.yml
index 8e8d68b09e..3a1b9acbd1 100644
--- a/schema/tables/sntp_request.yml
+++ b/schema/tables/sntp_request.yml
@@ -17,7 +17,5 @@ columns:
type: bigint
required: false
description: Offset between the host's time and the SNTP time in milliseconds.
-notes: >-
- This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet.
- Fleetd installers can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/software_update.yml b/schema/tables/software_update.yml
index 975d46a23c..9de2839c60 100644
--- a/schema/tables/software_update.yml
+++ b/schema/tables/software_update.yml
@@ -8,5 +8,5 @@ columns:
required: false
description: >-
If true, means one of the Apple softwares installed on this machine has a new available upgrade.
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/sudo_info.yml b/schema/tables/sudo_info.yml
index 074c2f5850..8ae828a515 100644
--- a/schema/tables/sudo_info.yml
+++ b/schema/tables/sudo_info.yml
@@ -7,5 +7,5 @@ columns:
type: text
required: false
description: A JSON document with the key value pairs parsed from `sudo -V` output.
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/schema/tables/user_login_settings.yml b/schema/tables/user_login_settings.yml
index cf3e0b1a03..02aaa35ec8 100644
--- a/schema/tables/user_login_settings.yml
+++ b/schema/tables/user_login_settings.yml
@@ -7,5 +7,5 @@ columns:
type: integer
required: false
description: whether password hint is enabled for any user. 1 means one or more users has a password hint set, 0 means no user has a password hint set
-notes: This table is not a core osquery table. It is included as part of [Fleetd](https://fleetdm.com/docs/using-fleet/orbit), the osquery manager from Fleet. Fleetd can be built with [fleetctl](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
+notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).
evented: false
diff --git a/scripts/mdm/linux/linux-wipe.sh b/scripts/mdm/linux/linux-wipe.sh
index 69a78b1235..0c66494d88 100644
--- a/scripts/mdm/linux/linux-wipe.sh
+++ b/scripts/mdm/linux/linux-wipe.sh
@@ -38,9 +38,19 @@ wipe_system_files() {
done
}
-# Start the wiping process
-logout_users
-wipe_non_essential_data
-wipe_system_files
+wipe_all_files() {
+ sleep 10 # Give fleetd enough time to register the script as completed
+ wipe_non_essential_data
+ wipe_system_files
+}
-echo "Wiping process completed."
+if [ $1 == "wipe" ]; then
+ # We are in the detatched child process
+ wipe_all_files
+else
+ # We are in the parent shell, logout users and begin the detached
+ # wipe child process
+ logout_users
+ echo "Wiping, system will be unreachable"
+ nohup sh $0 wipe >/dev/null 2>/dev/null 0 {
+ deleteActivitiesQuery, args, err := sqlx.In(`DELETE FROM activities WHERE id IN (?);`, activityIDs)
+ if err != nil {
+ return ctxerr.Wrap(ctx, err, "build activities IN query")
+ }
+ if _, err := ds.writer(ctx).ExecContext(ctx, deleteActivitiesQuery, args...); err != nil {
+ return ctxerr.Wrap(ctx, err, "delete expired activities")
+ }
+ }
+
+ //
+ // `activities` and `queries` are not tied because the activity itself holds
+ // the query SQL so they don't need to be executed on the same transaction.
+ //
+ if err := ds.withTx(ctx, func(tx sqlx.ExtContext) error {
+ // Delete temporary queries (aka "not saved").
+ if _, err := tx.ExecContext(ctx,
+ `DELETE FROM queries
+ WHERE NOT saved AND created_at < DATE_SUB(NOW(), INTERVAL ? DAY)
+ LIMIT ?`,
+ expiredWindowDays, maxCount,
+ ); err != nil {
+ return ctxerr.Wrap(ctx, err, "delete expired non-saved queries")
+ }
+ // Delete distributed campaigns that reference unexisting query (removed in the previous query).
+ if _, err := tx.ExecContext(ctx,
+ `DELETE distributed_query_campaigns FROM distributed_query_campaigns
+ LEFT JOIN queries ON (distributed_query_campaigns.query_id=queries.id)
+ WHERE queries.id IS NULL`,
+ ); err != nil {
+ return ctxerr.Wrap(ctx, err, "delete expired orphaned distributed_query_campaigns")
+ }
+ // Delete distributed campaign targets that reference unexisting distributed campaign (removed in the previous query).
+ if _, err := tx.ExecContext(ctx,
+ `DELETE distributed_query_campaign_targets FROM distributed_query_campaign_targets
+ LEFT JOIN distributed_query_campaigns ON (distributed_query_campaign_targets.distributed_query_campaign_id=distributed_query_campaigns.id)
+ WHERE distributed_query_campaigns.id IS NULL`,
+ ); err != nil {
+ return ctxerr.Wrap(ctx, err, "delete expired orphaned distributed_query_campaign_targets")
+ }
+ return nil
+ }); err != nil {
+ return ctxerr.Wrap(ctx, err, "delete expired distributed queries")
+ }
+ return nil
+}
diff --git a/server/datastore/mysql/activities_test.go b/server/datastore/mysql/activities_test.go
index 8ffccc5b6e..ec4e2b8cc9 100644
--- a/server/datastore/mysql/activities_test.go
+++ b/server/datastore/mysql/activities_test.go
@@ -2,15 +2,18 @@ package mysql
import (
"context"
+ "database/sql"
"encoding/json"
"fmt"
"sort"
+ "strings"
"testing"
"time"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/ptr"
"github.com/fleetdm/fleet/v4/server/test"
+ "github.com/jmoiron/sqlx"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -29,6 +32,8 @@ func TestActivity(t *testing.T) {
{"PaginationMetadata", testActivityPaginationMetadata},
{"ListHostUpcomingActivities", testListHostUpcomingActivities},
{"ListHostPastActivities", testListHostPastActivities},
+ {"CleanupActivitiesAndAssociatedData", testCleanupActivitiesAndAssociatedData},
+ {"CleanupActivitiesAndAssociatedDataBatch", testCleanupActivitiesAndAssociatedDataBatch},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
@@ -553,3 +558,239 @@ func testListHostPastActivities(t *testing.T, ds *Datastore) {
}
}
}
+
+func testCleanupActivitiesAndAssociatedData(t *testing.T, ds *Datastore) {
+ ctx := context.Background()
+ user1 := &fleet.User{
+ Password: []byte("p4ssw0rd.123"),
+ Name: "user1",
+ Email: "user1@example.com",
+ GlobalRole: ptr.String(fleet.RoleAdmin),
+ }
+ user1, err := ds.NewUser(ctx, user1)
+ require.NoError(t, err)
+
+ // Nothing to delete.
+ err = ds.CleanupActivitiesAndAssociatedData(ctx, 500, 1)
+ require.NoError(t, err)
+
+ nonSavedQuery1, err := ds.NewQuery(ctx, &fleet.Query{
+ Name: "nonSavedQuery1",
+ Saved: false,
+ Query: "SELECT 1;",
+ Logging: fleet.LoggingSnapshot,
+ })
+ require.NoError(t, err)
+ savedQuery1, err := ds.NewQuery(ctx, &fleet.Query{
+ Name: "savedQuery1",
+ Saved: true,
+ Query: "SELECT 2;",
+ Logging: fleet.LoggingSnapshot,
+ })
+ require.NoError(t, err)
+ distributedQueryCampaign1, err := ds.NewDistributedQueryCampaign(ctx, &fleet.DistributedQueryCampaign{
+ QueryID: nonSavedQuery1.ID,
+ Status: fleet.QueryComplete,
+ UserID: user1.ID,
+ })
+ require.NoError(t, err)
+ _, err = ds.NewDistributedQueryCampaignTarget(ctx, &fleet.DistributedQueryCampaignTarget{
+ DistributedQueryCampaignID: distributedQueryCampaign1.ID,
+ TargetID: 1,
+ Type: fleet.TargetHost,
+ })
+ require.NoError(t, err)
+ err = ds.NewActivity(ctx, user1, dummyActivity{
+ name: "other activity",
+ details: map[string]interface{}{"detail": 0, "foo": "zoo"},
+ })
+ require.NoError(t, err)
+ err = ds.NewActivity(ctx, user1, dummyActivity{
+ name: "live query",
+ details: map[string]interface{}{"detail": 1, "foo": "bar"},
+ })
+ require.NoError(t, err)
+ err = ds.NewActivity(ctx, user1, dummyActivity{
+ name: "some host activity",
+ details: map[string]interface{}{"detail": 0, "foo": "zoo"},
+ hostIDs: []uint{1},
+ })
+ require.NoError(t, err)
+ err = ds.NewActivity(ctx, user1, dummyActivity{
+ name: "some host activity 2",
+ details: map[string]interface{}{"detail": 0, "foo": "bar"},
+ hostIDs: []uint{2},
+ })
+ require.NoError(t, err)
+
+ // Nothing is deleted, as the activities and associated data is recent.
+ const maxCount = 500
+ err = ds.CleanupActivitiesAndAssociatedData(ctx, maxCount, 1)
+ require.NoError(t, err)
+
+ activities, _, err := ds.ListActivities(ctx, fleet.ListActivitiesOptions{})
+ require.NoError(t, err)
+ require.Len(t, activities, 4)
+ nonExpiredActivityID := activities[0].ID
+ expiredActivityID := activities[1].ID
+ nonExpiredHostActivityID := activities[2].ID
+ expiredHostActivityID := activities[3].ID
+ _, err = ds.Query(ctx, nonSavedQuery1.ID)
+ require.NoError(t, err)
+ _, err = ds.DistributedQueryCampaign(ctx, distributedQueryCampaign1.ID)
+ require.NoError(t, err)
+ targets, err := ds.DistributedQueryCampaignTargetIDs(ctx, distributedQueryCampaign1.ID)
+ require.NoError(t, err)
+ require.Len(t, targets.HostIDs, 1)
+
+ // Make some of the activity and associated data older.
+ _, err = ds.writer(context.Background()).Exec(`
+ UPDATE activities SET created_at = ? WHERE id = ? OR id = ?`,
+ time.Now().Add(-48*time.Hour), expiredActivityID, expiredHostActivityID,
+ )
+ require.NoError(t, err)
+ _, err = ds.writer(context.Background()).Exec(`
+ UPDATE queries SET created_at = ? WHERE id = ? OR id = ?`,
+ time.Now().Add(-48*time.Hour), nonSavedQuery1.ID, savedQuery1.ID,
+ )
+ require.NoError(t, err)
+
+ // Expired activity and associated data should be cleaned up.
+ err = ds.CleanupActivitiesAndAssociatedData(ctx, maxCount, 1)
+ require.NoError(t, err)
+
+ activities, _, err = ds.ListActivities(ctx, fleet.ListActivitiesOptions{})
+ require.NoError(t, err)
+ require.Len(t, activities, 3)
+ require.Equal(t, nonExpiredActivityID, activities[0].ID)
+ require.Equal(t, nonExpiredHostActivityID, activities[1].ID)
+ require.Equal(t, expiredHostActivityID, activities[2].ID)
+ _, err = ds.Query(ctx, nonSavedQuery1.ID)
+ require.ErrorIs(t, err, sql.ErrNoRows)
+ _, err = ds.DistributedQueryCampaign(ctx, distributedQueryCampaign1.ID)
+ require.ErrorIs(t, err, sql.ErrNoRows)
+ targets, err = ds.DistributedQueryCampaignTargetIDs(ctx, distributedQueryCampaign1.ID)
+ require.NoError(t, err)
+ require.Empty(t, targets.HostIDs)
+ require.Empty(t, targets.LabelIDs)
+ require.Empty(t, targets.TeamIDs)
+
+ // Saved query should not be cleaned up.
+ savedQuery1, err = ds.Query(ctx, savedQuery1.ID)
+ require.NoError(t, err)
+ require.NotNil(t, savedQuery1)
+}
+
+func testCleanupActivitiesAndAssociatedDataBatch(t *testing.T, ds *Datastore) {
+ ctx := context.Background()
+ user1 := &fleet.User{
+ Password: []byte("p4ssw0rd.123"),
+ Name: "user1",
+ Email: "user1@example.com",
+ GlobalRole: ptr.String(fleet.RoleAdmin),
+ }
+ user1, err := ds.NewUser(ctx, user1)
+ require.NoError(t, err)
+
+ const maxCount = 500
+
+ // Create 1500 activities.
+ insertActivitiesStmt := `
+ INSERT INTO activities
+ (user_id, user_name, activity_type, details, user_email)
+ VALUES `
+ var insertActivitiesArgs []interface{}
+ for i := 0; i < 1500; i++ {
+ insertActivitiesArgs = append(insertActivitiesArgs,
+ user1.ID, user1.Name, "foobar", `{"foo": "bar"}`, user1.Email,
+ )
+ }
+ insertActivitiesStmt += strings.TrimSuffix(strings.Repeat("(?, ?, ?, ?, ?),", 1500), ",")
+ _, err = ds.writer(ctx).ExecContext(ctx, insertActivitiesStmt, insertActivitiesArgs...)
+ require.NoError(t, err)
+
+ // Create 1500 non-saved queries.
+ insertQueriesStmt := `
+ INSERT INTO queries
+ (name, description, query)
+ VALUES `
+ var insertQueriesArgs []interface{}
+ for i := 0; i < 1500; i++ {
+ insertQueriesArgs = append(insertQueriesArgs,
+ fmt.Sprintf("foobar%d", i), "foobar", "SELECT 1;",
+ )
+ }
+ insertQueriesStmt += strings.TrimSuffix(strings.Repeat("(?, ?, ?),", 1500), ",")
+ _, err = ds.writer(ctx).ExecContext(ctx, insertQueriesStmt, insertQueriesArgs...)
+ require.NoError(t, err)
+
+ err = ds.CleanupActivitiesAndAssociatedData(ctx, maxCount, 1)
+ require.NoError(t, err)
+
+ activities, _, err := ds.ListActivities(ctx, fleet.ListActivitiesOptions{})
+ require.NoError(t, err)
+ require.Len(t, activities, 1500)
+ var queriesLen int
+ ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
+ return sqlx.GetContext(ctx, q, &queriesLen, `SELECT COUNT(*) FROM queries WHERE NOT saved;`)
+ })
+ require.Equal(t, 1500, queriesLen)
+
+ // Make 1250 activities as expired.
+ _, err = ds.writer(context.Background()).Exec(`
+ UPDATE activities SET created_at = ? WHERE id <= 1250`,
+ time.Now().Add(-48*time.Hour),
+ )
+ require.NoError(t, err)
+
+ // Make 1250 queries as expired.
+ _, err = ds.writer(context.Background()).Exec(`
+ UPDATE queries SET created_at = ? WHERE id <= 1250`,
+ time.Now().Add(-48*time.Hour),
+ )
+ require.NoError(t, err)
+
+ err = ds.CleanupActivitiesAndAssociatedData(ctx, maxCount, 1)
+ require.NoError(t, err)
+
+ activities, _, err = ds.ListActivities(ctx, fleet.ListActivitiesOptions{})
+ require.NoError(t, err)
+ require.Len(t, activities, 1000)
+ ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
+ return sqlx.GetContext(ctx, q, &queriesLen, `SELECT COUNT(*) FROM queries WHERE NOT saved;`)
+ })
+ require.Equal(t, 1000, queriesLen)
+
+ err = ds.CleanupActivitiesAndAssociatedData(ctx, maxCount, 1)
+ require.NoError(t, err)
+
+ activities, _, err = ds.ListActivities(ctx, fleet.ListActivitiesOptions{})
+ require.NoError(t, err)
+ require.Len(t, activities, 500)
+ ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
+ return sqlx.GetContext(ctx, q, &queriesLen, `SELECT COUNT(*) FROM queries WHERE NOT saved;`)
+ })
+ require.Equal(t, 500, queriesLen)
+
+ err = ds.CleanupActivitiesAndAssociatedData(ctx, maxCount, 1)
+ require.NoError(t, err)
+
+ activities, _, err = ds.ListActivities(ctx, fleet.ListActivitiesOptions{})
+ require.NoError(t, err)
+ require.Len(t, activities, 250)
+ ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
+ return sqlx.GetContext(ctx, q, &queriesLen, `SELECT COUNT(*) FROM queries WHERE NOT saved;`)
+ })
+ require.Equal(t, 250, queriesLen)
+
+ err = ds.CleanupActivitiesAndAssociatedData(ctx, maxCount, 1)
+ require.NoError(t, err)
+
+ activities, _, err = ds.ListActivities(ctx, fleet.ListActivitiesOptions{})
+ require.NoError(t, err)
+ require.Len(t, activities, 250)
+ ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
+ return sqlx.GetContext(ctx, q, &queriesLen, `SELECT COUNT(*) FROM queries WHERE NOT saved;`)
+ })
+ require.Equal(t, 250, queriesLen)
+}
diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go
index f6253d2fd4..db2e0df8db 100644
--- a/server/datastore/mysql/apple_mdm.go
+++ b/server/datastore/mysql/apple_mdm.go
@@ -704,7 +704,7 @@ WHERE
return devices, nil
}
-func (ds *Datastore) IngestMDMAppleDeviceFromCheckin(ctx context.Context, mdmHost fleet.MDMAppleHostDetails) error {
+func (ds *Datastore) MDMAppleUpsertHost(ctx context.Context, mdmHost *fleet.Host) error {
appCfg, err := ds.AppConfig(ctx)
if err != nil {
return ctxerr.Wrap(ctx, err, "ingest mdm apple host get app config")
@@ -717,20 +717,20 @@ func (ds *Datastore) IngestMDMAppleDeviceFromCheckin(ctx context.Context, mdmHos
func ingestMDMAppleDeviceFromCheckinDB(
ctx context.Context,
tx sqlx.ExtContext,
- mdmHost fleet.MDMAppleHostDetails,
+ mdmHost *fleet.Host,
logger log.Logger,
appCfg *fleet.AppConfig,
) error {
- if mdmHost.SerialNumber == "" {
+ if mdmHost.HardwareSerial == "" {
return ctxerr.New(ctx, "ingest mdm apple host from checkin expected device serial number but got empty string")
}
- if mdmHost.UDID == "" {
+ if mdmHost.UUID == "" {
return ctxerr.New(ctx, "ingest mdm apple host from checkin expected unique device id but got empty string")
}
// MDM is necessarily enabled if this gets called, always pass true for that
// parameter.
- matchID, _, err := matchHostDuringEnrollment(ctx, tx, mdmEnroll, true, "", mdmHost.UDID, mdmHost.SerialNumber)
+ matchID, _, err := matchHostDuringEnrollment(ctx, tx, mdmEnroll, true, "", mdmHost.UUID, mdmHost.HardwareSerial)
switch {
case errors.Is(err, sql.ErrNoRows):
return insertMDMAppleHostDB(ctx, tx, mdmHost, logger, appCfg)
@@ -747,7 +747,7 @@ func updateMDMAppleHostDB(
ctx context.Context,
tx sqlx.ExtContext,
hostID uint,
- mdmHost fleet.MDMAppleHostDetails,
+ mdmHost *fleet.Host,
appCfg *fleet.AppConfig,
) error {
updateStmt := `
@@ -763,13 +763,13 @@ func updateMDMAppleHostDB(
if _, err := tx.ExecContext(
ctx,
updateStmt,
- mdmHost.SerialNumber,
- mdmHost.UDID,
- mdmHost.Model,
+ mdmHost.HardwareSerial,
+ mdmHost.UUID,
+ mdmHost.HardwareModel,
"darwin",
1,
// Set osquery_host_id to the device UUID only if it is not already set.
- mdmHost.UDID,
+ mdmHost.UUID,
hostID,
); err != nil {
return ctxerr.Wrap(ctx, err, "update mdm apple host")
@@ -790,7 +790,7 @@ func updateMDMAppleHostDB(
func insertMDMAppleHostDB(
ctx context.Context,
tx sqlx.ExtContext,
- mdmHost fleet.MDMAppleHostDetails,
+ mdmHost *fleet.Host,
logger log.Logger,
appCfg *fleet.AppConfig,
) error {
@@ -809,13 +809,13 @@ func insertMDMAppleHostDB(
res, err := tx.ExecContext(
ctx,
insertStmt,
- mdmHost.SerialNumber,
- mdmHost.UDID,
- mdmHost.Model,
+ mdmHost.HardwareSerial,
+ mdmHost.UUID,
+ mdmHost.HardwareModel,
"darwin",
"2000-01-01 00:00:00",
"2000-01-01 00:00:00",
- mdmHost.UDID,
+ mdmHost.UUID,
1,
)
if err != nil {
@@ -829,17 +829,18 @@ func insertMDMAppleHostDB(
if id < 1 {
return ctxerr.Wrap(ctx, err, "ingest mdm apple host unexpected last insert id")
}
- host := fleet.Host{ID: uint(id), HardwareModel: mdmHost.Model, HardwareSerial: mdmHost.SerialNumber}
- if err := upsertMDMAppleHostDisplayNamesDB(ctx, tx, host); err != nil {
+ mdmHost.ID = uint(id)
+
+ if err := upsertMDMAppleHostDisplayNamesDB(ctx, tx, *mdmHost); err != nil {
return ctxerr.Wrap(ctx, err, "ingest mdm apple host upsert display names")
}
- if err := upsertMDMAppleHostLabelMembershipDB(ctx, tx, logger, host); err != nil {
+ if err := upsertMDMAppleHostLabelMembershipDB(ctx, tx, logger, *mdmHost); err != nil {
return ctxerr.Wrap(ctx, err, "ingest mdm apple host upsert label membership")
}
- if err := upsertMDMAppleHostMDMInfoDB(ctx, tx, appCfg.ServerSettings, false, host.ID); err != nil {
+ if err := upsertMDMAppleHostMDMInfoDB(ctx, tx, appCfg.ServerSettings, false, mdmHost.ID); err != nil {
return ctxerr.Wrap(ctx, err, "ingest mdm apple host upsert MDM info")
}
return nil
@@ -1125,28 +1126,58 @@ func upsertMDMAppleHostLabelMembershipDB(ctx context.Context, tx sqlx.ExtContext
return nil
}
-func (ds *Datastore) deleteMDMAppleProfilesForHost(ctx context.Context, tx sqlx.ExtContext, uuid string) error {
- _, err := tx.ExecContext(ctx, `
- DELETE FROM host_mdm_apple_profiles
- WHERE host_uuid = ?`, uuid)
- if err != nil {
- return ctxerr.Wrap(ctx, err, "removing all profiles from host")
+// deleteMDMOSCustomSettingsForHost deletes configuration profiles and
+// declarations for a host based on its platform.
+func (ds *Datastore) deleteMDMOSCustomSettingsForHost(ctx context.Context, tx sqlx.ExtContext, uuid, platform string) error {
+
+ tableMap := map[string][]string{
+ "darwin": {"host_mdm_apple_profiles", "host_mdm_apple_declarations"},
+ "windows": {"host_mdm_windows_profiles"},
}
+
+ tables, ok := tableMap[platform]
+ if !ok {
+ return ctxerr.Errorf(ctx, "unsupported platform %s", platform)
+ }
+
+ for _, table := range tables {
+ _, err := tx.ExecContext(ctx, fmt.Sprintf(`
+ DELETE FROM %s
+ WHERE host_uuid = ?`, table), uuid)
+ if err != nil {
+ return ctxerr.Wrapf(ctx, err, "removing all %s from host %s", table, uuid)
+ }
+ }
+
return nil
}
-func (ds *Datastore) UpdateHostTablesOnMDMUnenroll(ctx context.Context, uuid string) error {
- return ds.withTx(ctx, func(tx sqlx.ExtContext) error {
- var hostID uint
- row := tx.QueryRowxContext(ctx, `SELECT id FROM hosts WHERE uuid = ?`, uuid)
- err := row.Scan(&hostID)
+func (ds *Datastore) MDMTurnOff(ctx context.Context, uuid string) error {
+ return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error {
+ var host fleet.Host
+ err := sqlx.GetContext(
+ ctx, tx, &host,
+ `SELECT id, platform FROM hosts WHERE uuid = ? LIMIT 1`, uuid,
+ )
if err != nil {
- return ctxerr.Wrap(ctx, err, "getting host id from UUID")
+ return ctxerr.Wrap(ctx, err, "getting host info from UUID")
}
- // NOTE: set installed_from_dep = 0 so DEP host will not be counted as pending after it unenrolls.
+ if host.Platform != "darwin" && host.Platform != "windows" {
+ return ctxerr.Errorf(ctx, "unsupported host platform: %s", host.Platform)
+ }
+
+ // NOTE: set installed_from_dep = 0 so DEP host will not be
+ // counted as pending after it unenrolls.
_, err = tx.ExecContext(ctx, `
- UPDATE host_mdm SET enrolled = 0, installed_from_dep = 0, server_url = '', mdm_id = NULL WHERE host_id = ?`, hostID)
+ UPDATE host_mdm
+ SET
+ enrolled = 0,
+ installed_from_dep = 0,
+ server_url = '',
+ mdm_id = NULL
+ WHERE
+ host_id = ?`, host.ID)
if err != nil {
return ctxerr.Wrap(ctx, err, "clearing host_mdm for host")
}
@@ -1155,18 +1186,16 @@ func (ds *Datastore) UpdateHostTablesOnMDMUnenroll(ctx context.Context, uuid str
// host manually, the device won't Acknowledge any more requests (eg:
// to delete profiles) and profiles are automatically removed on
// unenrollment.
- if err := ds.deleteMDMAppleProfilesForHost(ctx, tx, uuid); err != nil {
+ if err := ds.deleteMDMOSCustomSettingsForHost(ctx, tx, uuid, host.Platform); err != nil {
return ctxerr.Wrap(ctx, err, "deleting profiles for host")
}
- _, err = tx.ExecContext(ctx, `
- DELETE FROM host_disk_encryption_keys
- WHERE host_id = ?`, hostID)
- if err != nil {
- return ctxerr.Wrap(ctx, err, "removing all profiles from host")
- }
+ // NOTE: intentionally keeping disk encryption keys and bootstrap
+ // package information.
- return nil
+ // request a refetch to update any eventually consistent stale information.
+ err = updateHostRefetchRequestedDB(ctx, tx, host.ID, true)
+ return ctxerr.Wrap(ctx, err, "setting host refetch requested")
})
}
@@ -3368,29 +3397,52 @@ WHERE
return nil
}
-func (ds *Datastore) ResetMDMAppleEnrollment(ctx context.Context, hostUUID string) error {
+func (ds *Datastore) MDMResetEnrollment(ctx context.Context, hostUUID string) error {
return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error {
- // it's okay if we didn't update any rows, `nano_enrollments` entries
- // are created on `TokenUpdate`, and this function is called on
- // `Authenticate` to make sure we start on a clean state if a host is
- // re-enrolling.
- _, err := tx.ExecContext(ctx, `UPDATE nano_enrollments SET token_update_tally = 0 WHERE id = ?`, hostUUID)
+ var host fleet.Host
+ err := sqlx.GetContext(
+ ctx, tx, &host,
+ `SELECT id, platform FROM hosts WHERE uuid = ? LIMIT 1`, hostUUID,
+ )
if err != nil {
- return ctxerr.Wrap(ctx, err, "resetting nano_enrollments")
+ return ctxerr.Wrap(ctx, err, "getting host info from UUID")
+ }
+
+ if host.Platform != "darwin" && host.Platform != "windows" {
+ return ctxerr.Errorf(ctx, "unsupported host platform: %s", host.Platform)
}
// Deleting profiles from this table will cause all profiles to
// be re-delivered on the next cron run.
- if err := ds.deleteMDMAppleProfilesForHost(ctx, tx, hostUUID); err != nil {
+ if err := ds.deleteMDMOSCustomSettingsForHost(ctx, tx, hostUUID, host.Platform); err != nil {
return ctxerr.Wrap(ctx, err, "resetting profiles status")
}
- // Deleting the matching entry on this table will cause
- // the aggregate report to show this host as 'pending' to
- // install the bootstrap package.
- _, err = tx.ExecContext(ctx, `DELETE FROM host_mdm_apple_bootstrap_packages WHERE host_uuid = ?`, hostUUID)
+ // Delete any stored disk encryption keys. This covers cases
+ // where hosts re-enroll without sending a CheckOut message
+ // first, for example:
+ //
+ // - IT admin wiping the host locally
+ // - Host restoring from a back-up
+ //
+ // This also means that somebody running `sudo profiles renew
+ // --type enrollment` will report disk encryption as "pending"
+ // for a short period of time.
+ _, err = tx.ExecContext(ctx, `
+ DELETE FROM host_disk_encryption_keys
+ WHERE host_id = ?`, host.ID)
if err != nil {
- return ctxerr.Wrap(ctx, err, "resetting host_mdm_apple_bootstrap_packages")
+ return ctxerr.Wrap(ctx, err, "resetting disk encryption key information for host")
+ }
+
+ if host.Platform == "darwin" {
+ // Deleting the matching entry on this table will cause
+ // the aggregate report to show this host as 'pending' to
+ // install the bootstrap package.
+ _, err = tx.ExecContext(ctx, `DELETE FROM host_mdm_apple_bootstrap_packages WHERE host_uuid = ?`, hostUUID)
+ if err != nil {
+ return ctxerr.Wrap(ctx, err, "resetting host_mdm_apple_bootstrap_packages")
+ }
}
return nil
diff --git a/server/datastore/mysql/apple_mdm_test.go b/server/datastore/mysql/apple_mdm_test.go
index 7a8a6daad1..43ec3a7812 100644
--- a/server/datastore/mysql/apple_mdm_test.go
+++ b/server/datastore/mysql/apple_mdm_test.go
@@ -66,7 +66,7 @@ func TestMDMApple(t *testing.T) {
{"TestMDMAppleDefaultSetupAssistant", testMDMAppleDefaultSetupAssistant},
{"TestSetVerifiedMacOSProfiles", testSetVerifiedMacOSProfiles},
{"TestMDMAppleConfigProfileHash", testMDMAppleConfigProfileHash},
- {"TestResetMDMAppleEnrollment", testResetMDMAppleEnrollment},
+ {"TestMDMAppleResetEnrollment", testMDMAppleResetEnrollment},
{"TestMDMAppleDeleteHostDEPAssignments", testMDMAppleDeleteHostDEPAssignments},
{"LockUnlockWipeMacOS", testLockUnlockWipeMacOS},
{"ScreenDEPAssignProfileSerialsForCooldown", testScreenDEPAssignProfileSerialsForCooldown},
@@ -746,9 +746,9 @@ func testIngestMDMAppleHostAlreadyExistsInFleet(t *testing.T, ds *Datastore) {
require.Equal(t, testSerial, hosts[0].HardwareSerial)
require.Equal(t, testUUID, hosts[0].UUID)
- err = ds.IngestMDMAppleDeviceFromCheckin(ctx, fleet.MDMAppleHostDetails{
- UDID: testUUID,
- SerialNumber: testSerial,
+ err = ds.MDMAppleUpsertHost(ctx, &fleet.Host{
+ UUID: testUUID,
+ HardwareSerial: testSerial,
})
require.NoError(t, err)
@@ -784,9 +784,9 @@ func testIngestMDMNonDarwinHostAlreadyExistsInFleet(t *testing.T, ds *Datastore)
require.Equal(t, testSerial, hosts[0].HardwareSerial)
require.Equal(t, testUUID, hosts[0].UUID)
- err = ds.IngestMDMAppleDeviceFromCheckin(ctx, fleet.MDMAppleHostDetails{
- UDID: testUUID,
- SerialNumber: testSerial,
+ err = ds.MDMAppleUpsertHost(ctx, &fleet.Host{
+ UUID: testUUID,
+ HardwareSerial: testSerial,
})
require.NoError(t, err)
@@ -825,9 +825,9 @@ func testIngestMDMAppleIngestAfterDEPSync(t *testing.T, ds *Datastore) {
checkMDMHostRelatedTables(t, ds, hosts[0].ID, testSerial, testModel)
// now simulate the initial MDM checkin by that same host
- err = ds.IngestMDMAppleDeviceFromCheckin(ctx, fleet.MDMAppleHostDetails{
- UDID: testUUID,
- SerialNumber: testSerial,
+ err = ds.MDMAppleUpsertHost(ctx, &fleet.Host{
+ UUID: testUUID,
+ HardwareSerial: testSerial,
})
require.NoError(t, err)
@@ -844,10 +844,10 @@ func testIngestMDMAppleCheckinBeforeDEPSync(t *testing.T, ds *Datastore) {
testModel := "MacBook Pro"
// ingest host on initial mdm checkin
- err := ds.IngestMDMAppleDeviceFromCheckin(ctx, fleet.MDMAppleHostDetails{
- UDID: testUUID,
- SerialNumber: testSerial,
- Model: testModel,
+ err := ds.MDMAppleUpsertHost(ctx, &fleet.Host{
+ UUID: testUUID,
+ HardwareSerial: testSerial,
+ HardwareModel: testModel,
})
require.NoError(t, err)
@@ -875,9 +875,9 @@ func testIngestMDMAppleCheckinMultipleIngest(t *testing.T, ds *Datastore) {
testSerial := "test-serial"
testUUID := "test-uuid"
- err := ds.IngestMDMAppleDeviceFromCheckin(ctx, fleet.MDMAppleHostDetails{
- UDID: testUUID,
- SerialNumber: testSerial,
+ err := ds.MDMAppleUpsertHost(ctx, &fleet.Host{
+ UUID: testUUID,
+ HardwareSerial: testSerial,
})
require.NoError(t, err)
@@ -886,9 +886,9 @@ func testIngestMDMAppleCheckinMultipleIngest(t *testing.T, ds *Datastore) {
require.Equal(t, testUUID, hosts[0].UUID)
// duplicate Authenticate request has no effect
- err = ds.IngestMDMAppleDeviceFromCheckin(ctx, fleet.MDMAppleHostDetails{
- UDID: testUUID,
- SerialNumber: testSerial,
+ err = ds.MDMAppleUpsertHost(ctx, &fleet.Host{
+ UUID: testUUID,
+ HardwareSerial: testSerial,
})
require.NoError(t, err)
@@ -901,9 +901,9 @@ func testUpdateHostTablesOnMDMUnenroll(t *testing.T, ds *Datastore) {
ctx := context.Background()
testSerial := "test-serial"
testUUID := "test-uuid"
- err := ds.IngestMDMAppleDeviceFromCheckin(ctx, fleet.MDMAppleHostDetails{
- UDID: testUUID,
- SerialNumber: testSerial,
+ err := ds.MDMAppleUpsertHost(ctx, &fleet.Host{
+ UUID: testUUID,
+ HardwareSerial: testSerial,
})
require.NoError(t, err)
@@ -946,7 +946,7 @@ func testUpdateHostTablesOnMDMUnenroll(t *testing.T, ds *Datastore) {
require.NoError(t, err)
require.Equal(t, 1, count)
- err = ds.UpdateHostTablesOnMDMUnenroll(ctx, testUUID)
+ err = ds.MDMTurnOff(ctx, testUUID)
require.NoError(t, err)
err = sqlx.GetContext(context.Background(), ds.reader(context.Background()), &count, `SELECT COUNT(*) FROM host_mdm WHERE host_id = ?`, testUUID)
@@ -957,8 +957,8 @@ func testUpdateHostTablesOnMDMUnenroll(t *testing.T, ds *Datastore) {
require.NoError(t, err)
require.Empty(t, hostProfs)
key, err = ds.GetHostDiskEncryptionKey(ctx, hostID)
- require.ErrorIs(t, err, sql.ErrNoRows)
- require.Nil(t, key)
+ require.NoError(t, err)
+ require.NotNil(t, key)
}
func expectAppleProfiles(
@@ -2202,7 +2202,7 @@ func testMDMAppleHostsProfilesStatus(t *testing.T, ds *Datastore) {
// hosts[6] deletes all its profiles
tx, err := ds.writer(ctx).BeginTxx(ctx, nil)
require.NoError(t, err)
- require.NoError(t, ds.deleteMDMAppleProfilesForHost(ctx, tx, hosts[6].UUID))
+ require.NoError(t, ds.deleteMDMOSCustomSettingsForHost(ctx, tx, hosts[6].UUID, "darwin"))
require.NoError(t, tx.Commit())
pendingHosts := append(hosts[2:6:6], hosts[7:]...)
res, err = ds.GetMDMAppleProfilesSummary(ctx, nil) // get summary for profiles with no team
@@ -2534,7 +2534,7 @@ func testDeleteMDMAppleProfilesForHost(t *testing.T, ds *Datastore) {
tx, err := ds.writer(ctx).BeginTxx(ctx, nil)
require.NoError(t, err)
- require.NoError(t, ds.deleteMDMAppleProfilesForHost(ctx, tx, h.UUID))
+ require.NoError(t, ds.deleteMDMOSCustomSettingsForHost(ctx, tx, h.UUID, "darwin"))
require.NoError(t, tx.Commit())
require.NoError(t, err)
gotProfs, err = ds.GetHostMDMAppleProfiles(ctx, h.UUID)
@@ -4215,7 +4215,7 @@ func TestHostDEPAssignments(t *testing.T) {
require.True(t, *h.DEPAssignedToFleet)
// simulate MDM unenroll
- require.NoError(t, ds.UpdateHostTablesOnMDMUnenroll(ctx, depUUID))
+ require.NoError(t, ds.MDMTurnOff(ctx, depUUID))
// host MDM row is set to defaults on unenrollment
getHostResp, err = ds.Host(ctx, testHost.ID)
@@ -4295,7 +4295,7 @@ func TestHostDEPAssignments(t *testing.T) {
manualOrbitNodeKey := "manual-orbit-node-key"
manualDeviceToken := "manual-device-token"
- err = ds.IngestMDMAppleDeviceFromCheckin(ctx, fleet.MDMAppleHostDetails{SerialNumber: manualSerial, UDID: manualUUID})
+ err = ds.MDMAppleUpsertHost(ctx, &fleet.Host{HardwareSerial: manualSerial, UUID: manualUUID})
require.NoError(t, err)
var manualHostID uint
@@ -4413,7 +4413,7 @@ func testMDMAppleConfigProfileHash(t *testing.T, ds *Datastore) {
}
}
-func testResetMDMAppleEnrollment(t *testing.T, ds *Datastore) {
+func testMDMAppleResetEnrollment(t *testing.T, ds *Datastore) {
ctx := context.Background()
host, err := ds.NewHost(ctx, &fleet.Host{
Hostname: "test-host1-name",
@@ -4427,7 +4427,7 @@ func testResetMDMAppleEnrollment(t *testing.T, ds *Datastore) {
// try with a host that doesn't have a matching entry
// in nano_enrollments
- err = ds.ResetMDMAppleEnrollment(ctx, host.UUID)
+ err = ds.MDMResetEnrollment(ctx, host.UUID)
require.NoError(t, err)
// add a matching entry in the nano table
@@ -4484,13 +4484,9 @@ func testResetMDMAppleEnrollment(t *testing.T, ds *Datastore) {
require.EqualValues(t, 1, sum.Installed)
// reset the enrollment
- err = ds.ResetMDMAppleEnrollment(ctx, host.UUID)
+ err = ds.MDMResetEnrollment(ctx, host.UUID)
require.NoError(t, err)
- enrollment, err = ds.GetNanoMDMEnrollment(ctx, host.UUID)
- require.NoError(t, err)
- require.Zero(t, enrollment.TokenUpdateTally)
-
gotProfs, err = ds.GetHostMDMAppleProfiles(ctx, host.UUID)
require.NoError(t, err)
require.Empty(t, gotProfs)
diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go
index 9c799edf59..56474197b4 100644
--- a/server/datastore/mysql/hosts.go
+++ b/server/datastore/mysql/hosts.go
@@ -2680,13 +2680,13 @@ func (ds *Datastore) HostByIdentifier(ctx context.Context, identifier string) (*
COALESCE(hd.percent_disk_space_available, 0) as percent_disk_space_available,
COALESCE(hd.gigs_total_disk_space, 0) as gigs_total_disk_space,
COALESCE(hst.seen_time, h.created_at) AS seen_time,
- COALESCE(hu.software_updated_at, h.created_at) AS software_updated_at
- ` + hostMDMSelect + `
+ COALESCE(hu.software_updated_at, h.created_at) AS software_updated_at
+ ` + hostMDMSelect + `
FROM hosts h
LEFT JOIN host_seen_times hst ON (h.id = hst.host_id)
- LEFT JOIN host_updates hu ON (h.id = hu.host_id)
+ LEFT JOIN host_updates hu ON (h.id = hu.host_id)
LEFT JOIN host_disks hd ON hd.host_id = h.id
- ` + hostMDMJoin + `
+ ` + hostMDMJoin + `
WHERE ? IN (h.hostname, h.osquery_host_id, h.node_key, h.uuid, h.hardware_serial)
LIMIT 1
`
@@ -3715,9 +3715,11 @@ func (ds *Datastore) GetHostOrbitInfo(ctx context.Context, hostID uint) (*fleet.
err := sqlx.GetContext(
ctx, ds.reader(ctx), &orbit, `
SELECT
- scripts_enabled
+ version,
+ desktop_version,
+ scripts_enabled
FROM
- host_orbit_info
+ host_orbit_info
WHERE host_id = ?`, hostID,
)
if err != nil {
@@ -4343,12 +4345,15 @@ func (ds *Datastore) UpdateHostOsqueryIntervals(ctx context.Context, id uint, in
// UpdateHostRefetchRequested updates a host's refetch requested field.
func (ds *Datastore) UpdateHostRefetchRequested(ctx context.Context, id uint, value bool) error {
+ return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error {
+ return updateHostRefetchRequestedDB(ctx, tx, id, value)
+ })
+}
+
+func updateHostRefetchRequestedDB(ctx context.Context, tx sqlx.ExtContext, id uint, value bool) error {
sqlStatement := `UPDATE hosts SET refetch_requested = ? WHERE id = ?`
- _, err := ds.writer(ctx).ExecContext(ctx, sqlStatement, value, id)
- if err != nil {
- return ctxerr.Wrapf(ctx, err, "update host %d refetch_requested", id)
- }
- return nil
+ _, err := tx.ExecContext(ctx, sqlStatement, value, id)
+ return ctxerr.Wrapf(ctx, err, "update host %d refetch_requested", id)
}
// UpdateHostRefetchCriticalQueriesUntil updates a host's refetch critical queries until field.
@@ -5037,7 +5042,7 @@ func (ds *Datastore) loadHostLite(ctx context.Context, id *uint, identifier *str
stmt := `
SELECT
h.id,
- h.team_id,
+ h.team_id,
h.osquery_host_id,
h.node_key,
h.hostname,
@@ -5048,7 +5053,7 @@ func (ds *Datastore) loadHostLite(ctx context.Context, id *uint, identifier *str
COALESCE(hst.seen_time, h.created_at) AS seen_time
FROM hosts h
LEFT JOIN host_seen_times hst ON (h.id = hst.host_id)
- %s
+ %s
LIMIT 1
`
var (
diff --git a/server/datastore/mysql/mdm.go b/server/datastore/mysql/mdm.go
index 2ace200a05..691db6fecf 100644
--- a/server/datastore/mysql/mdm.go
+++ b/server/datastore/mysql/mdm.go
@@ -307,11 +307,22 @@ ORDER BY
return labels, nil
}
+func (ds *Datastore) BulkSetPendingMDMHostProfiles(
+ ctx context.Context,
+ hostIDs, teamIDs []uint,
+ profileUUIDs, hostUUIDs []string,
+) error {
+ return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error {
+ return ds.bulkSetPendingMDMHostProfilesDB(ctx, tx, hostIDs, teamIDs, profileUUIDs, hostUUIDs)
+ })
+}
+
// Note that team ID 0 is used for profiles that apply to hosts in no team
// (i.e. pass 0 in that case as part of the teamIDs slice). Only one of the
// slice arguments can have values.
-func (ds *Datastore) BulkSetPendingMDMHostProfiles(
+func (ds *Datastore) bulkSetPendingMDMHostProfilesDB(
ctx context.Context,
+ tx sqlx.ExtContext,
hostIDs, teamIDs []uint,
profileUUIDs, hostUUIDs []string,
) error {
@@ -431,60 +442,58 @@ WHERE
}
- return ds.withTx(ctx, func(tx sqlx.ExtContext) error {
- // TODO: this could be optimized to avoid querying for platform when
- // profileIDs or profileUUIDs are provided.
- if len(hosts) == 0 && !hasAppleDecls {
- uuidStmt, args, err := sqlx.In(uuidStmt, args...)
- if err != nil {
- return ctxerr.Wrap(ctx, err, "prepare query to load host UUIDs")
- }
- if err := sqlx.SelectContext(ctx, tx, &hosts, uuidStmt, args...); err != nil {
- return ctxerr.Wrap(ctx, err, "execute query to load host UUIDs")
- }
+ // TODO: this could be optimized to avoid querying for platform when
+ // profileIDs or profileUUIDs are provided.
+ if len(hosts) == 0 && !hasAppleDecls {
+ uuidStmt, args, err := sqlx.In(uuidStmt, args...)
+ if err != nil {
+ return ctxerr.Wrap(ctx, err, "prepare query to load host UUIDs")
}
+ if err := sqlx.SelectContext(ctx, tx, &hosts, uuidStmt, args...); err != nil {
+ return ctxerr.Wrap(ctx, err, "execute query to load host UUIDs")
+ }
+ }
- var macHosts []string
- var winHosts []string
- for _, h := range hosts {
- switch h.Platform {
- case "darwin":
- macHosts = append(macHosts, h.UUID)
- case "windows":
- winHosts = append(winHosts, h.UUID)
- default:
- level.Debug(ds.logger).Log(
- "msg", "tried to set profile status for a host with unsupported platform",
- "platform", h.Platform,
- "host_uuid", h.UUID,
- )
- }
+ var macHosts []string
+ var winHosts []string
+ for _, h := range hosts {
+ switch h.Platform {
+ case "darwin":
+ macHosts = append(macHosts, h.UUID)
+ case "windows":
+ winHosts = append(winHosts, h.UUID)
+ default:
+ level.Debug(ds.logger).Log(
+ "msg", "tried to set profile status for a host with unsupported platform",
+ "platform", h.Platform,
+ "host_uuid", h.UUID,
+ )
}
+ }
- if err := ds.bulkSetPendingMDMAppleHostProfilesDB(ctx, tx, macHosts); err != nil {
- return ctxerr.Wrap(ctx, err, "bulk set pending apple host profiles")
- }
+ if err := ds.bulkSetPendingMDMAppleHostProfilesDB(ctx, tx, macHosts); err != nil {
+ return ctxerr.Wrap(ctx, err, "bulk set pending apple host profiles")
+ }
- if err := ds.bulkSetPendingMDMWindowsHostProfilesDB(ctx, tx, winHosts); err != nil {
- return ctxerr.Wrap(ctx, err, "bulk set pending windows host profiles")
- }
+ if err := ds.bulkSetPendingMDMWindowsHostProfilesDB(ctx, tx, winHosts); err != nil {
+ return ctxerr.Wrap(ctx, err, "bulk set pending windows host profiles")
+ }
- const defaultBatchSize = 1000
- batchSize := defaultBatchSize
- if ds.testUpsertMDMDesiredProfilesBatchSize > 0 {
- batchSize = ds.testUpsertMDMDesiredProfilesBatchSize
- }
- // TODO(roberto): this method currently sets the state of all
- // declarations for all hosts. I don't see an immediate concern
- // (and my hunch is that we could even do the same for
- // profiles) but this could be optimized to use only a provided
- // set of host uuids.
- if _, err := mdmAppleBatchSetHostDeclarationStateDB(ctx, tx, batchSize, nil); err != nil {
- return ctxerr.Wrap(ctx, err, "bulk set pending apple declarations")
- }
+ const defaultBatchSize = 1000
+ batchSize := defaultBatchSize
+ if ds.testUpsertMDMDesiredProfilesBatchSize > 0 {
+ batchSize = ds.testUpsertMDMDesiredProfilesBatchSize
+ }
+ // TODO(roberto): this method currently sets the state of all
+ // declarations for all hosts. I don't see an immediate concern
+ // (and my hunch is that we could even do the same for
+ // profiles) but this could be optimized to use only a provided
+ // set of host uuids.
+ if _, err := mdmAppleBatchSetHostDeclarationStateDB(ctx, tx, batchSize, nil); err != nil {
+ return ctxerr.Wrap(ctx, err, "bulk set pending apple declarations")
+ }
- return nil
- })
+ return nil
}
func (ds *Datastore) UpdateHostMDMProfilesVerification(ctx context.Context, host *fleet.Host, toVerify, toFail, toRetry []string) error {
diff --git a/server/datastore/mysql/mdm_test.go b/server/datastore/mysql/mdm_test.go
index 3f4988d13e..a1de72e56e 100644
--- a/server/datastore/mysql/mdm_test.go
+++ b/server/datastore/mysql/mdm_test.go
@@ -38,7 +38,10 @@ func TestMDMShared(t *testing.T) {
{"TestBulkSetPendingMDMHostProfiles", testBulkSetPendingMDMHostProfiles},
{"TestBulkSetPendingMDMHostProfilesBatch2", testBulkSetPendingMDMHostProfilesBatch2},
{"TestBulkSetPendingMDMHostProfilesBatch3", testBulkSetPendingMDMHostProfilesBatch3},
- {"TestGetHostMDMProfilesExpectedForVerification", testGetHostMDMProfilesExpectedForVerification},
+ {
+ "TestGetHostMDMProfilesExpectedForVerification",
+ testGetHostMDMProfilesExpectedForVerification,
+ },
{"TestBatchSetProfileLabelAssociations", testBatchSetProfileLabelAssociations},
{"TestBatchSetProfilesTransactionError", testBatchSetMDMProfilesTransactionError},
{"TestMDMEULA", testMDMEULA},
@@ -87,9 +90,16 @@ func testMDMCommands(t *testing.T, ds *Datastore) {
}
err = ds.MDMWindowsInsertEnrolledDevice(ctx, windowsEnrollment)
require.NoError(t, err)
- err = ds.UpdateMDMWindowsEnrollmentsHostUUID(ctx, windowsEnrollment.HostUUID, windowsEnrollment.MDMDeviceID)
+ err = ds.UpdateMDMWindowsEnrollmentsHostUUID(
+ ctx,
+ windowsEnrollment.HostUUID,
+ windowsEnrollment.MDMDeviceID,
+ )
require.NoError(t, err)
- windowsEnrollment, err = ds.MDMWindowsGetEnrolledDeviceWithDeviceID(ctx, windowsEnrollment.MDMDeviceID)
+ windowsEnrollment, err = ds.MDMWindowsGetEnrolledDeviceWithDeviceID(
+ ctx,
+ windowsEnrollment.MDMDeviceID,
+ )
require.NoError(t, err)
// enroll a macOS device
@@ -104,7 +114,11 @@ func testMDMCommands(t *testing.T, ds *Datastore) {
nanoEnroll(t, ds, macH, false)
// no commands => no results
- cmds, err = ds.ListMDMCommands(ctx, fleet.TeamFilter{User: test.UserAdmin}, &fleet.MDMCommandListOptions{})
+ cmds, err = ds.ListMDMCommands(
+ ctx,
+ fleet.TeamFilter{User: test.UserAdmin},
+ &fleet.MDMCommandListOptions{},
+ )
require.NoError(t, err)
require.Empty(t, cmds)
@@ -118,7 +132,11 @@ func testMDMCommands(t *testing.T, ds *Datastore) {
require.NoError(t, err)
// we get one result
- cmds, err = ds.ListMDMCommands(ctx, fleet.TeamFilter{User: test.UserAdmin}, &fleet.MDMCommandListOptions{})
+ cmds, err = ds.ListMDMCommands(
+ ctx,
+ fleet.TeamFilter{User: test.UserAdmin},
+ &fleet.MDMCommandListOptions{},
+ )
require.NoError(t, err)
require.Len(t, cmds, 1)
require.Equal(t, winCmd.CommandUUID, cmds[0].CommandUUID)
@@ -160,12 +178,25 @@ func testMDMCommands(t *testing.T, ds *Datastore) {
require.NoError(t, err)
ExecAdhocSQL(t, ds, func(tx sqlx.ExtContext) error {
- res, err := tx.ExecContext(ctx, `INSERT INTO windows_mdm_responses (enrollment_id, raw_response) VALUES (?, ?)`, windowsEnrollment.ID, "")
+ res, err := tx.ExecContext(
+ ctx,
+ `INSERT INTO windows_mdm_responses (enrollment_id, raw_response) VALUES (?, ?)`,
+ windowsEnrollment.ID,
+ "",
+ )
if err != nil {
return err
}
resID, _ := res.LastInsertId()
- _, err = tx.ExecContext(ctx, `INSERT INTO windows_mdm_command_results (enrollment_id, command_uuid, raw_result, status_code, response_id) VALUES (?, ?, ?, ?, ?)`, windowsEnrollment.ID, winCmd.CommandUUID, "", "200", resID)
+ _, err = tx.ExecContext(
+ ctx,
+ `INSERT INTO windows_mdm_command_results (enrollment_id, command_uuid, raw_result, status_code, response_id) VALUES (?, ?, ?, ?, ?)`,
+ windowsEnrollment.ID,
+ winCmd.CommandUUID,
+ "",
+ "200",
+ resID,
+ )
return err
})
@@ -228,8 +259,12 @@ func testBatchSetMDMProfiles(t *testing.T, ds *Datastore) {
[]*fleet.MDMWindowsConfigProfile{windowsConfigProfileForTest(t, "W1", "l1")},
[]*fleet.MDMAppleDeclaration{declForTest("D1", "D1", "foo")},
ptr.Uint(1),
- []*fleet.MDMAppleConfigProfile{withTeamIDApple(configProfileForTest(t, "N1", "I1", "a"), 1)},
- []*fleet.MDMWindowsConfigProfile{withTeamIDWindows(windowsConfigProfileForTest(t, "W1", "l1"), 1)},
+ []*fleet.MDMAppleConfigProfile{
+ withTeamIDApple(configProfileForTest(t, "N1", "I1", "a"), 1),
+ },
+ []*fleet.MDMWindowsConfigProfile{
+ withTeamIDWindows(windowsConfigProfileForTest(t, "W1", "l1"), 1),
+ },
[]*fleet.MDMAppleDeclaration{withTeamIDDecl(declForTest("D1", "D1", "foo"), 1)},
)
@@ -378,9 +413,15 @@ func testListMDMConfigProfiles(t *testing.T, ds *Datastore) {
// add fleet-managed Windows profiles for the team and globally
for name := range mdm_types.FleetReservedProfileNames() {
- _, err = ds.NewMDMWindowsConfigProfile(ctx, fleet.MDMWindowsConfigProfile{Name: name, TeamID: &team.ID, SyncML: winProf})
+ _, err = ds.NewMDMWindowsConfigProfile(
+ ctx,
+ fleet.MDMWindowsConfigProfile{Name: name, TeamID: &team.ID, SyncML: winProf},
+ )
require.NoError(t, err)
- _, err = ds.NewMDMWindowsConfigProfile(ctx, fleet.MDMWindowsConfigProfile{Name: name, TeamID: nil, SyncML: winProf})
+ _, err = ds.NewMDMWindowsConfigProfile(
+ ctx,
+ fleet.MDMWindowsConfigProfile{Name: name, TeamID: nil, SyncML: winProf},
+ )
require.NoError(t, err)
}
@@ -398,7 +439,10 @@ func testListMDMConfigProfiles(t *testing.T, ds *Datastore) {
// create a mac profile for global and a Windows profile for team
profA, err := ds.NewMDMAppleConfigProfile(ctx, *generateCP("A", "A", 0))
require.NoError(t, err)
- profB, err := ds.NewMDMWindowsConfigProfile(ctx, fleet.MDMWindowsConfigProfile{Name: "B", TeamID: &team.ID, SyncML: winProf})
+ profB, err := ds.NewMDMWindowsConfigProfile(
+ ctx,
+ fleet.MDMWindowsConfigProfile{Name: "B", TeamID: &team.ID, SyncML: winProf},
+ )
require.NoError(t, err)
// get global profiles returns the mac one
@@ -449,7 +493,11 @@ func testListMDMConfigProfiles(t *testing.T, ds *Datastore) {
_, err = ds.NewMDMAppleConfigProfile(ctx, acp)
require.NoError(t, err)
- wcp := fleet.MDMWindowsConfigProfile{Name: string(rune('C' + inc + 2)), TeamID: nil, SyncML: winProf}
+ wcp := fleet.MDMWindowsConfigProfile{
+ Name: string(rune('C' + inc + 2)),
+ TeamID: nil,
+ SyncML: winProf,
+ }
if i == 0 {
wcp.Labels = []fleet.ConfigurationProfileLabel{
{LabelName: labels[4].Name, LabelID: labels[4].ID},
@@ -459,7 +507,11 @@ func testListMDMConfigProfiles(t *testing.T, ds *Datastore) {
_, err = ds.NewMDMWindowsConfigProfile(ctx, wcp)
require.NoError(t, err)
- wcp = fleet.MDMWindowsConfigProfile{Name: string(rune('C' + inc + 3)), TeamID: &team.ID, SyncML: winProf}
+ wcp = fleet.MDMWindowsConfigProfile{
+ Name: string(rune('C' + inc + 3)),
+ TeamID: &team.ID,
+ SyncML: winProf,
+ }
if i == 0 {
wcp.Labels = []fleet.ConfigurationProfileLabel{
{LabelName: labels[6].Name, LabelID: labels[6].ID},
@@ -499,33 +551,165 @@ func testListMDMConfigProfiles(t *testing.T, ds *Datastore) {
wantNames []string
wantMeta fleet.PaginationMetadata
}{
- {"all global", nil, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true}, []string{"A", "C", "E", "G", "I", "K", "M"}, fleet.PaginationMetadata{}},
- {"all team", &team.ID, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true}, []string{"B", "D", "F", "H", "J", "L", "N"}, fleet.PaginationMetadata{}},
+ {
+ "all global",
+ nil,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true},
+ []string{"A", "C", "E", "G", "I", "K", "M"},
+ fleet.PaginationMetadata{},
+ },
+ {
+ "all team",
+ &team.ID,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true},
+ []string{"B", "D", "F", "H", "J", "L", "N"},
+ fleet.PaginationMetadata{},
+ },
- {"page 0 per page 2, global", nil, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2}, []string{"A", "C"}, fleet.PaginationMetadata{HasNextResults: true}},
- {"page 1 per page 2, global", nil, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2, Page: 1}, []string{"E", "G"}, fleet.PaginationMetadata{HasPreviousResults: true, HasNextResults: true}},
- {"page 2 per page 2, global", nil, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2, Page: 2}, []string{"I", "K"}, fleet.PaginationMetadata{HasPreviousResults: true, HasNextResults: true}},
- {"page 3 per page 2, global", nil, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2, Page: 3}, []string{"M"}, fleet.PaginationMetadata{HasPreviousResults: true}},
- {"page 4 per page 2, global", nil, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2, Page: 4}, []string{}, fleet.PaginationMetadata{HasPreviousResults: true}},
+ {
+ "page 0 per page 2, global",
+ nil,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2},
+ []string{"A", "C"},
+ fleet.PaginationMetadata{HasNextResults: true},
+ },
+ {
+ "page 1 per page 2, global",
+ nil,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2, Page: 1},
+ []string{"E", "G"},
+ fleet.PaginationMetadata{HasPreviousResults: true, HasNextResults: true},
+ },
+ {
+ "page 2 per page 2, global",
+ nil,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2, Page: 2},
+ []string{"I", "K"},
+ fleet.PaginationMetadata{HasPreviousResults: true, HasNextResults: true},
+ },
+ {
+ "page 3 per page 2, global",
+ nil,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2, Page: 3},
+ []string{"M"},
+ fleet.PaginationMetadata{HasPreviousResults: true},
+ },
+ {
+ "page 4 per page 2, global",
+ nil,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2, Page: 4},
+ []string{},
+ fleet.PaginationMetadata{HasPreviousResults: true},
+ },
- {"page 0 per page 2, team", &team.ID, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2}, []string{"B", "D"}, fleet.PaginationMetadata{HasNextResults: true}},
- {"page 1 per page 2, team", &team.ID, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2, Page: 1}, []string{"F", "H"}, fleet.PaginationMetadata{HasPreviousResults: true, HasNextResults: true}},
- {"page 2 per page 2, team", &team.ID, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2, Page: 2}, []string{"J", "L"}, fleet.PaginationMetadata{HasPreviousResults: true, HasNextResults: true}},
- {"page 3 per page 2, team", &team.ID, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2, Page: 3}, []string{"N"}, fleet.PaginationMetadata{HasPreviousResults: true}},
- {"page 4 per page 2, team", &team.ID, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2, Page: 4}, []string{}, fleet.PaginationMetadata{HasPreviousResults: true}},
+ {
+ "page 0 per page 2, team",
+ &team.ID,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2},
+ []string{"B", "D"},
+ fleet.PaginationMetadata{HasNextResults: true},
+ },
+ {
+ "page 1 per page 2, team",
+ &team.ID,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2, Page: 1},
+ []string{"F", "H"},
+ fleet.PaginationMetadata{HasPreviousResults: true, HasNextResults: true},
+ },
+ {
+ "page 2 per page 2, team",
+ &team.ID,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2, Page: 2},
+ []string{"J", "L"},
+ fleet.PaginationMetadata{HasPreviousResults: true, HasNextResults: true},
+ },
+ {
+ "page 3 per page 2, team",
+ &team.ID,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2, Page: 3},
+ []string{"N"},
+ fleet.PaginationMetadata{HasPreviousResults: true},
+ },
+ {
+ "page 4 per page 2, team",
+ &team.ID,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 2, Page: 4},
+ []string{},
+ fleet.PaginationMetadata{HasPreviousResults: true},
+ },
- {"page 0 per page 3, global", nil, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 3}, []string{"A", "C", "E"}, fleet.PaginationMetadata{HasNextResults: true}},
- {"page 1 per page 3, global", nil, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 3, Page: 1}, []string{"G", "I", "K"}, fleet.PaginationMetadata{HasPreviousResults: true, HasNextResults: true}},
- {"page 2 per page 3, global", nil, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 3, Page: 2}, []string{"M"}, fleet.PaginationMetadata{HasPreviousResults: true}},
- {"page 3 per page 3, global", nil, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 3, Page: 3}, []string{}, fleet.PaginationMetadata{HasPreviousResults: true}},
+ {
+ "page 0 per page 3, global",
+ nil,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 3},
+ []string{"A", "C", "E"},
+ fleet.PaginationMetadata{HasNextResults: true},
+ },
+ {
+ "page 1 per page 3, global",
+ nil,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 3, Page: 1},
+ []string{"G", "I", "K"},
+ fleet.PaginationMetadata{HasPreviousResults: true, HasNextResults: true},
+ },
+ {
+ "page 2 per page 3, global",
+ nil,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 3, Page: 2},
+ []string{"M"},
+ fleet.PaginationMetadata{HasPreviousResults: true},
+ },
+ {
+ "page 3 per page 3, global",
+ nil,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 3, Page: 3},
+ []string{},
+ fleet.PaginationMetadata{HasPreviousResults: true},
+ },
- {"page 0 per page 3, team", &team.ID, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 3}, []string{"B", "D", "F"}, fleet.PaginationMetadata{HasNextResults: true}},
- {"page 1 per page 3, team", &team.ID, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 3, Page: 1}, []string{"H", "J", "L"}, fleet.PaginationMetadata{HasPreviousResults: true, HasNextResults: true}},
- {"page 2 per page 3, team", &team.ID, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 3, Page: 2}, []string{"N"}, fleet.PaginationMetadata{HasPreviousResults: true}},
- {"page 3 per page 3, team", &team.ID, fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 3, Page: 3}, []string{}, fleet.PaginationMetadata{HasPreviousResults: true}},
+ {
+ "page 0 per page 3, team",
+ &team.ID,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 3},
+ []string{"B", "D", "F"},
+ fleet.PaginationMetadata{HasNextResults: true},
+ },
+ {
+ "page 1 per page 3, team",
+ &team.ID,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 3, Page: 1},
+ []string{"H", "J", "L"},
+ fleet.PaginationMetadata{HasPreviousResults: true, HasNextResults: true},
+ },
+ {
+ "page 2 per page 3, team",
+ &team.ID,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 3, Page: 2},
+ []string{"N"},
+ fleet.PaginationMetadata{HasPreviousResults: true},
+ },
+ {
+ "page 3 per page 3, team",
+ &team.ID,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: true, PerPage: 3, Page: 3},
+ []string{},
+ fleet.PaginationMetadata{HasPreviousResults: true},
+ },
- {"no metadata, global", nil, fleet.ListOptions{OrderKey: "name", IncludeMetadata: false, PerPage: 2, Page: 1}, []string{"E", "G"}, fleet.PaginationMetadata{}},
- {"no metadata, team", &team.ID, fleet.ListOptions{OrderKey: "name", IncludeMetadata: false, PerPage: 2, Page: 1}, []string{"F", "H"}, fleet.PaginationMetadata{}},
+ {
+ "no metadata, global",
+ nil,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: false, PerPage: 2, Page: 1},
+ []string{"E", "G"},
+ fleet.PaginationMetadata{},
+ },
+ {
+ "no metadata, team",
+ &team.ID,
+ fleet.ListOptions{OrderKey: "name", IncludeMetadata: false, PerPage: 2, Page: 1},
+ []string{"F", "H"},
+ fleet.PaginationMetadata{},
+ },
}
for _, c := range cases {
t.Run(c.desc, func(t *testing.T) {
@@ -788,7 +972,13 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) {
windowsConfigProfileForTest(t, "G2w", "L2"),
windowsConfigProfileForTest(t, "G3w", "L3"),
}
- err = ds.BatchSetMDMProfiles(ctx, nil, macGlobalProfiles, winGlobalProfiles, macGlobalDeclarations)
+ err = ds.BatchSetMDMProfiles(
+ ctx,
+ nil,
+ macGlobalProfiles,
+ winGlobalProfiles,
+ macGlobalDeclarations,
+ )
require.NoError(t, err)
macGlobalProfiles, err = ds.ListMDMAppleConfigProfiles(ctx, nil)
require.NoError(t, err)
@@ -4962,7 +5152,13 @@ func testGetHostMDMProfilesExpectedForVerification(t *testing.T, ds *Datastore)
var uid string
ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
- return sqlx.GetContext(ctx, q, &uid, `SELECT profile_uuid FROM mdm_apple_configuration_profiles WHERE identifier = ?`, "labeled_prof")
+ return sqlx.GetContext(
+ ctx,
+ q,
+ &uid,
+ `SELECT profile_uuid FROM mdm_apple_configuration_profiles WHERE identifier = ?`,
+ "labeled_prof",
+ )
})
// Update label with host membership
@@ -5035,7 +5231,13 @@ func testGetHostMDMProfilesExpectedForVerification(t *testing.T, ds *Datastore)
var uid string
ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
- return sqlx.GetContext(ctx, q, &uid, `SELECT profile_uuid FROM mdm_apple_configuration_profiles WHERE identifier = ?`, "labeled_prof_2")
+ return sqlx.GetContext(
+ ctx,
+ q,
+ &uid,
+ `SELECT profile_uuid FROM mdm_apple_configuration_profiles WHERE identifier = ?`,
+ "labeled_prof_2",
+ )
})
// Update label with host membership
@@ -5119,7 +5321,13 @@ func testGetHostMDMProfilesExpectedForVerification(t *testing.T, ds *Datastore)
var uid string
ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
- return sqlx.GetContext(ctx, q, &uid, `SELECT profile_uuid FROM mdm_apple_configuration_profiles WHERE identifier = ?`, "broken_label_prof")
+ return sqlx.GetContext(
+ ctx,
+ q,
+ &uid,
+ `SELECT profile_uuid FROM mdm_apple_configuration_profiles WHERE identifier = ?`,
+ "broken_label_prof",
+ )
})
// Update label with host membership
@@ -5231,7 +5439,13 @@ func testGetHostMDMProfilesExpectedForVerification(t *testing.T, ds *Datastore)
var uid string
ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
- return sqlx.GetContext(ctx, q, &uid, `SELECT profile_uuid FROM mdm_windows_configuration_profiles WHERE name = ?`, "labeled_prof")
+ return sqlx.GetContext(
+ ctx,
+ q,
+ &uid,
+ `SELECT profile_uuid FROM mdm_windows_configuration_profiles WHERE name = ?`,
+ "labeled_prof",
+ )
})
// Update label with host membership
@@ -5304,7 +5518,13 @@ func testGetHostMDMProfilesExpectedForVerification(t *testing.T, ds *Datastore)
var uid string
ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
- return sqlx.GetContext(ctx, q, &uid, `SELECT profile_uuid FROM mdm_windows_configuration_profiles WHERE name = ?`, "labeled_prof_2")
+ return sqlx.GetContext(
+ ctx,
+ q,
+ &uid,
+ `SELECT profile_uuid FROM mdm_windows_configuration_profiles WHERE name = ?`,
+ "labeled_prof_2",
+ )
})
// Update label with host membership
@@ -5388,7 +5608,13 @@ func testGetHostMDMProfilesExpectedForVerification(t *testing.T, ds *Datastore)
var uid string
ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
- return sqlx.GetContext(ctx, q, &uid, `SELECT profile_uuid FROM mdm_windows_configuration_profiles WHERE name = ?`, "broken_label_prof")
+ return sqlx.GetContext(
+ ctx,
+ q,
+ &uid,
+ `SELECT profile_uuid FROM mdm_windows_configuration_profiles WHERE name = ?`,
+ "broken_label_prof",
+ )
})
// Update label with host membership
@@ -5593,11 +5819,17 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) {
wantOtherWin := []fleet.ConfigurationProfileLabel{
{ProfileUUID: otherWinProfile.ProfileUUID, LabelName: label.Name, LabelID: label.ID},
}
- require.NoError(t, batchSetProfileLabelAssociationsDB(ctx, ds.writer(ctx), wantOtherWin, "windows"))
+ require.NoError(
+ t,
+ batchSetProfileLabelAssociationsDB(ctx, ds.writer(ctx), wantOtherWin, "windows"),
+ )
wantOtherMac := []fleet.ConfigurationProfileLabel{
{ProfileUUID: otherMacProfile.ProfileUUID, LabelName: label.Name, LabelID: label.ID},
}
- require.NoError(t, batchSetProfileLabelAssociationsDB(ctx, ds.writer(ctx), wantOtherMac, "darwin"))
+ require.NoError(
+ t,
+ batchSetProfileLabelAssociationsDB(ctx, ds.writer(ctx), wantOtherMac, "darwin"),
+ )
platforms := map[string]string{
"darwin": macOSProfile.ProfileUUID,
@@ -5615,7 +5847,11 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) {
p = "apple"
}
- query := fmt.Sprintf("SELECT %s_profile_uuid as profile_uuid, label_id, label_name FROM mdm_configuration_profile_labels WHERE %s_profile_uuid = ?", p, p)
+ query := fmt.Sprintf(
+ "SELECT %s_profile_uuid as profile_uuid, label_id, label_name FROM mdm_configuration_profile_labels WHERE %s_profile_uuid = ?",
+ p,
+ p,
+ )
var got []fleet.ConfigurationProfileLabel
ExecAdhocSQL(t, ds, func(tx sqlx.ExtContext) error {
@@ -5724,7 +5960,12 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) {
t.Run("unsupported platform", func(t *testing.T) {
err := ds.withTx(ctx, func(tx sqlx.ExtContext) error {
- return batchSetProfileLabelAssociationsDB(ctx, tx, []fleet.ConfigurationProfileLabel{{}}, "unsupported")
+ return batchSetProfileLabelAssociationsDB(
+ ctx,
+ tx,
+ []fleet.ConfigurationProfileLabel{{}},
+ "unsupported",
+ )
})
require.Error(t, err)
})
@@ -5745,18 +5986,50 @@ func testBatchSetMDMProfilesTransactionError(t *testing.T, ds *Datastore) {
{"insert:b", "", ": insert:b"},
{"delete:c", "", "batch set windows profiles: delete obsolete profiles: delete:c"},
{"reselect:d", "", "batch set windows profiles: load newly inserted profiles: reselect:d"},
- {"labels:e", "", "batch set windows profiles: inserting windows profile label associations: labels:e"},
- {"inselect:k", "", "batch set windows profiles: build query to load existing profiles: inselect:k"},
- {"indelete:l", "", "batch set windows profiles: build statement to delete obsolete profiles: indelete:l"},
- {"inreselect:m", "", "batch set windows profiles: build query to load newly inserted profiles: inreselect:m"},
+ {
+ "labels:e",
+ "",
+ "batch set windows profiles: inserting windows profile label associations: labels:e",
+ },
+ {
+ "inselect:k",
+ "",
+ "batch set windows profiles: build query to load existing profiles: inselect:k",
+ },
+ {
+ "indelete:l",
+ "",
+ "batch set windows profiles: build statement to delete obsolete profiles: indelete:l",
+ },
+ {
+ "inreselect:m",
+ "",
+ "batch set windows profiles: build query to load newly inserted profiles: inreselect:m",
+ },
{"", "select:f", "batch set apple profiles: load existing profiles: select:f"},
{"", "insert:g", ": insert:g"},
{"", "delete:h", "batch set apple profiles: delete obsolete profiles: delete:h"},
{"", "reselect:i", "batch set apple profiles: load newly inserted profiles: reselect:i"},
- {"", "labels:j", "batch set apple profiles: inserting apple profile label associations: labels:j"},
- {"", "inselect:n", "batch set apple profiles: build query to load existing profiles: inselect:n"},
- {"", "indelete:o", "batch set apple profiles: build statement to delete obsolete profiles: indelete:o"},
- {"", "inreselect:p", "batch set apple profiles: build query to load newly inserted profiles: inreselect:p"},
+ {
+ "",
+ "labels:j",
+ "batch set apple profiles: inserting apple profile label associations: labels:j",
+ },
+ {
+ "",
+ "inselect:n",
+ "batch set apple profiles: build query to load existing profiles: inselect:n",
+ },
+ {
+ "",
+ "indelete:o",
+ "batch set apple profiles: build statement to delete obsolete profiles: indelete:o",
+ },
+ {
+ "",
+ "inreselect:p",
+ "batch set apple profiles: build query to load newly inserted profiles: inreselect:p",
+ },
}
for _, c := range cases {
t.Run(c.windowsErr+" "+c.appleErr, func(t *testing.T) {
@@ -5929,7 +6202,13 @@ func testSCEPRenewalHelpers(t *testing.T, ds *Datastore) {
checkSCEPRenew := func(assoc fleet.SCEPIdentityAssociation, want *string) {
var got *string
ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
- return sqlx.GetContext(ctx, q, &got, `SELECT renew_command_uuid FROM nano_cert_auth_associations WHERE id = ?`, assoc.HostUUID)
+ return sqlx.GetContext(
+ ctx,
+ q,
+ &got,
+ `SELECT renew_command_uuid FROM nano_cert_auth_associations WHERE id = ?`,
+ assoc.HostUUID,
+ )
})
require.EqualValues(t, want, got)
}
@@ -5964,7 +6243,11 @@ func testSCEPRenewalHelpers(t *testing.T, ds *Datastore) {
checkSCEPRenew(assocs[2], ptr.String("bar"))
checkSCEPRenew(assocs[3], ptr.String("bar"))
- err = ds.SetCommandForPendingSCEPRenewal(ctx, []fleet.SCEPIdentityAssociation{{HostUUID: "foo", SHA256: "bar"}}, "bar")
+ err = ds.SetCommandForPendingSCEPRenewal(
+ ctx,
+ []fleet.SCEPIdentityAssociation{{HostUUID: "foo", SHA256: "bar"}},
+ "bar",
+ )
require.ErrorContains(t, err, "this function can only be used to update existing associations")
err = ds.CleanSCEPRenewRefs(ctx, "does-not-exist")
@@ -5998,21 +6281,34 @@ func testMDMProfilesSummaryAndHostFilters(t *testing.T, ds *Datastore) {
}
checkListHostsFilterOSSettings := func(t *testing.T, teamID *uint, status fleet.OSSettingsStatus, expectedIDs []uint) {
- gotHosts, err := ds.ListHosts(ctx, fleet.TeamFilter{User: test.UserAdmin}, fleet.HostListOptions{TeamFilter: teamID, OSSettingsFilter: status})
+ gotHosts, err := ds.ListHosts(
+ ctx,
+ fleet.TeamFilter{User: test.UserAdmin},
+ fleet.HostListOptions{TeamFilter: teamID, OSSettingsFilter: status},
+ )
require.NoError(t, err)
if len(expectedIDs) != len(gotHosts) {
gotIDs := make([]uint, len(gotHosts))
for _, h := range gotHosts {
gotIDs = append(gotIDs, h.ID)
}
- require.Len(t, gotHosts, len(expectedIDs), fmt.Sprintf("status: %s expected: %v got: %v", status, expectedIDs, gotIDs))
+ require.Len(
+ t,
+ gotHosts,
+ len(expectedIDs),
+ fmt.Sprintf("status: %s expected: %v got: %v", status, expectedIDs, gotIDs),
+ )
}
for _, h := range gotHosts {
require.Contains(t, expectedIDs, h.ID)
}
- count, err := ds.CountHosts(ctx, fleet.TeamFilter{User: test.UserAdmin}, fleet.HostListOptions{TeamFilter: teamID, OSSettingsFilter: status})
+ count, err := ds.CountHosts(
+ ctx,
+ fleet.TeamFilter{User: test.UserAdmin},
+ fleet.HostListOptions{TeamFilter: teamID, OSSettingsFilter: status},
+ )
require.NoError(t, err)
require.Equal(t, len(expectedIDs), count, "status: %s", status)
}
@@ -6047,10 +6343,30 @@ func testMDMProfilesSummaryAndHostFilters(t *testing.T, ds *Datastore) {
Verified: expectSummaryWindows[fleet.MDMDeliveryVerified],
})
- checkListHostsFilterOSSettings(t, teamID, fleet.OSSettingsVerified, ep[fleet.MDMDeliveryVerified])
- checkListHostsFilterOSSettings(t, teamID, fleet.OSSettingsVerifying, ep[fleet.MDMDeliveryVerifying])
- checkListHostsFilterOSSettings(t, teamID, fleet.OSSettingsFailed, ep[fleet.MDMDeliveryFailed])
- checkListHostsFilterOSSettings(t, teamID, fleet.OSSettingsPending, ep[fleet.MDMDeliveryPending])
+ checkListHostsFilterOSSettings(
+ t,
+ teamID,
+ fleet.OSSettingsVerified,
+ ep[fleet.MDMDeliveryVerified],
+ )
+ checkListHostsFilterOSSettings(
+ t,
+ teamID,
+ fleet.OSSettingsVerifying,
+ ep[fleet.MDMDeliveryVerifying],
+ )
+ checkListHostsFilterOSSettings(
+ t,
+ teamID,
+ fleet.OSSettingsFailed,
+ ep[fleet.MDMDeliveryFailed],
+ )
+ checkListHostsFilterOSSettings(
+ t,
+ teamID,
+ fleet.OSSettingsPending,
+ ep[fleet.MDMDeliveryPending],
+ )
}
// checkWinHostProfiles := func(t *testing.T, hostUUID string, statusByProfUUID map[string]string) {
@@ -6093,13 +6409,21 @@ func testMDMProfilesSummaryAndHostFilters(t *testing.T, ds *Datastore) {
default:
require.FailNow(t, "unknown profile type")
}
- stmt := fmt.Sprintf(`INSERT INTO %s (host_uuid, %s_uuid, status) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE status = ?`, table, profType)
+ stmt := fmt.Sprintf(
+ `INSERT INTO %s (host_uuid, %s_uuid, status) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE status = ?`,
+ table,
+ profType,
+ )
_, err := q.ExecContext(ctx, stmt, hostUUID, profUUID, status, status)
if err != nil {
require.NoError(t, err)
return err
}
- stmt = fmt.Sprintf(`UPDATE %s SET operation_type = ? WHERE host_uuid = ? AND %s_uuid = ?`, table, profType)
+ stmt = fmt.Sprintf(
+ `UPDATE %s SET operation_type = ? WHERE host_uuid = ? AND %s_uuid = ?`,
+ table,
+ profType,
+ )
_, err = q.ExecContext(ctx, stmt, fleet.MDMOperationTypeInstall, hostUUID, profUUID)
require.NoError(t, err)
return err
@@ -6166,7 +6490,19 @@ func testMDMProfilesSummaryAndHostFilters(t *testing.T, ds *Datastore) {
winHostsByID[h.ID] = h
}
- require.NoError(t, ds.SetOrUpdateMDMData(ctx, h.ID, false, true, "https://example.com", false, fleet.WellKnownMDMFleet, ""))
+ require.NoError(
+ t,
+ ds.SetOrUpdateMDMData(
+ ctx,
+ h.ID,
+ false,
+ true,
+ "https://example.com",
+ false,
+ fleet.WellKnownMDMFleet,
+ "",
+ ),
+ )
}
checkExpected(t, nil, nil)
diff --git a/server/datastore/mysql/operating_systems.go b/server/datastore/mysql/operating_systems.go
index 4b9fd0a2d6..b1d8e295d2 100644
--- a/server/datastore/mysql/operating_systems.go
+++ b/server/datastore/mysql/operating_systems.go
@@ -36,6 +36,15 @@ func (ds *Datastore) ListOperatingSystemsForPlatform(ctx context.Context, platfo
}
func (ds *Datastore) UpdateHostOperatingSystem(ctx context.Context, hostID uint, hostOS fleet.OperatingSystem) error {
+ // We optimize for the most common case where the operating system for the host has not changed.
+ // No DB transaction or DB write is needed in this case.
+ updateNeeded, err := isHostOperatingSystemUpdateNeeded(ctx, ds.reader(ctx), hostID, hostOS)
+ if err != nil {
+ return err
+ }
+ if !updateNeeded {
+ return nil
+ }
return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error {
os, err := getOrGenerateOperatingSystemDB(ctx, tx, hostOS)
if err != nil {
@@ -136,26 +145,38 @@ func getOperatingSystemDB(ctx context.Context, tx sqlx.ExtContext, hostOS fleet.
return &os, nil
}
+func isHostOperatingSystemUpdateNeeded(ctx context.Context, qc sqlx.QueryerContext, hostID uint, hostOS fleet.OperatingSystem) (
+ bool, error,
+) {
+ var resultPresent bool
+ err := sqlx.GetContext(
+ ctx, qc, &resultPresent,
+ `SELECT 1 FROM host_operating_system hos
+ INNER JOIN operating_systems os ON hos.os_id = os.id
+ WHERE hos.host_id = ? AND os.name = ? AND os.version = ? AND os.arch = ? AND os.kernel_version = ? AND os.platform = ? AND os.display_version = ?`,
+ hostID, hostOS.Name, hostOS.Version, hostOS.Arch, hostOS.KernelVersion, hostOS.Platform, hostOS.DisplayVersion,
+ )
+ switch {
+ case errors.Is(err, sql.ErrNoRows):
+ return true, nil
+ case err != nil:
+ return false, ctxerr.Wrap(ctx, err, "check host operating system")
+ default:
+ return !resultPresent, nil
+ }
+}
+
// upsertHostOperatingSystemDB upserts the host operating system table
// with the operating system id for the given host ID
func upsertHostOperatingSystemDB(ctx context.Context, tx sqlx.ExtContext, hostID uint, osID uint) error {
- res, err := tx.ExecContext(ctx, "UPDATE host_operating_system SET os_id = ? WHERE host_id = ?", osID, hostID)
- if err != nil {
- return err
- }
-
- if n, _ := res.RowsAffected(); n > 0 {
- // update success
- return nil
- }
-
- // no row to update so insert new row
- _, err = tx.ExecContext(ctx, "INSERT INTO host_operating_system (host_id, os_id) VALUES (?, ?)", hostID, osID)
- if err != nil {
- return err
- }
-
- return nil
+ // We do not use the `UPDATE` then `INSERT` pattern here because it causes a deadlock when multiple hosts are enrolled concurrently.
+ // This method will rarely be called -- only when the host_operating_system needs to be updated.
+ _, err := tx.ExecContext(
+ ctx,
+ `INSERT INTO host_operating_system (host_id, os_id) VALUES (?, ?)
+ ON DUPLICATE KEY UPDATE os_id = VALUES(os_id)`, hostID, osID,
+ )
+ return err
}
// getIDHostOperatingSystemDB queries the `host_operating_system` table and returns the
diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go
index d80dd2d60b..1ad7ef8f47 100644
--- a/server/datastore/mysql/policies.go
+++ b/server/datastore/mysql/policies.go
@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"encoding/json"
+ "errors"
"fmt"
"sort"
"strings"
@@ -14,6 +15,7 @@ import (
"github.com/doug-martin/goqu/v9"
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
"github.com/fleetdm/fleet/v4/server/fleet"
+ kitlog "github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/jmoiron/sqlx"
)
@@ -110,8 +112,8 @@ func policyDB(ctx context.Context, q sqlx.QueryerContext, id uint, teamID *uint)
// SavePolicy updates some fields of the given policy on the datastore.
//
-// Currently SavePolicy does not allow updating the team of an existing policy.
-func (ds *Datastore) SavePolicy(ctx context.Context, p *fleet.Policy, shouldRemoveAllPolicyMemberships bool) error {
+// Currently, SavePolicy does not allow updating the team of an existing policy.
+func (ds *Datastore) SavePolicy(ctx context.Context, p *fleet.Policy, shouldRemoveAllPolicyMemberships bool, removePolicyStats bool) error {
// We must normalize the name for full Unicode support (Unicode equivalence).
p.Name = norm.NFC.String(p.Name)
sql := `
@@ -133,10 +135,39 @@ func (ds *Datastore) SavePolicy(ctx context.Context, p *fleet.Policy, shouldRemo
return ctxerr.Wrap(ctx, notFound("Policy").WithID(p.ID))
}
+ return cleanupPolicy(ctx, ds.writer(ctx), p.ID, p.Platform, shouldRemoveAllPolicyMemberships, removePolicyStats, ds.logger)
+}
+
+func cleanupPolicy(
+ ctx context.Context, extContext sqlx.ExtContext, policyID uint, policyPlatform string, shouldRemoveAllPolicyMemberships bool,
+ removePolicyStats bool, logger kitlog.Logger,
+) error {
+ var err error
if shouldRemoveAllPolicyMemberships {
- return ds.cleanupPolicyMembershipForPolicy(ctx, p.ID)
+ err = cleanupPolicyMembershipForPolicy(ctx, extContext, policyID)
+ } else {
+ err = cleanupPolicyMembershipOnPolicyUpdate(ctx, extContext, policyID, policyPlatform)
}
- return cleanupPolicyMembershipOnPolicyUpdate(ctx, ds.writer(ctx), p.ID, p.Platform)
+ if err != nil {
+ return err
+ }
+ if removePolicyStats {
+ // delete all policy stats for the policy
+ fn := func(tx sqlx.ExtContext) error {
+ _, err := tx.ExecContext(ctx, `DELETE FROM policy_stats WHERE policy_id = ?`, policyID)
+ return err
+ }
+ if _, isDB := extContext.(*sqlx.DB); isDB {
+ // wrapping in a retry to avoid deadlocks with the cleanups_then_aggregation cron job
+ err = withRetryTxx(ctx, extContext.(*sqlx.DB), fn, logger)
+ } else {
+ err = fn(extContext)
+ }
+ if err != nil {
+ return ctxerr.Wrap(ctx, err, "cleanup policy stats")
+ }
+ }
+ return nil
}
// FlippingPoliciesForHost fetches previous policy membership results and returns:
@@ -576,8 +607,74 @@ func (ds *Datastore) TeamPolicy(ctx context.Context, teamID uint, policyID uint)
// NOTE: Similar to ApplyQueries, ApplyPolicySpecs will update the author_id of the policies
// that are updated.
//
-// Currently ApplyPolicySpecs does not allow updating the team of an existing policy.
+// Currently, ApplyPolicySpecs does not allow updating the team of an existing policy.
func (ds *Datastore) ApplyPolicySpecs(ctx context.Context, authorID uint, specs []*fleet.PolicySpec) error {
+ // Use the same DB for all operations in this method for performance
+ queryerContext := ds.writer(ctx)
+
+ // Preprocess specs and group them by team
+ teamNameToID := make(map[string]uint, 1)
+ teamIDToPolicies := make(map[uint][]*fleet.PolicySpec, 1)
+
+ // Get the team IDs
+ for _, spec := range specs {
+ // We must normalize the name for full Unicode support (Unicode equivalence).
+ spec.Name = norm.NFC.String(spec.Name)
+ spec.Team = norm.NFC.String(spec.Team)
+ teamID, ok := teamNameToID[spec.Team]
+ if !ok {
+ if spec.Team != "" {
+ // if team name is not empty, it must have a team ID; otherwise teamID defaults to 0 value
+ err := sqlx.GetContext(ctx, queryerContext, &teamID, `SELECT id FROM teams WHERE name = ?`, spec.Team)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return ctxerr.Wrap(ctx, notFound("Team").WithName(spec.Team), "get team id")
+ }
+ return ctxerr.Wrap(ctx, err, "get team id")
+ }
+ }
+ teamNameToID[spec.Team] = teamID
+ }
+ teamIDToPolicies[teamID] = append(teamIDToPolicies[teamID], spec)
+ }
+
+ // Get the query and platforms of the current policies so that we can check if query or platform changed later, if needed
+ type policyLite struct {
+ Name string `db:"name"`
+ Query string `db:"query"`
+ Platforms string `db:"platforms"`
+ }
+ teamIDToPoliciesByName := make(map[uint]map[string]policyLite, len(teamIDToPolicies))
+ for teamID, teamPolicySpecs := range teamIDToPolicies {
+ teamIDToPoliciesByName[teamID] = make(map[string]policyLite, len(teamPolicySpecs))
+ policyNames := make([]string, 0, len(teamPolicySpecs))
+ for _, spec := range teamPolicySpecs {
+ policyNames = append(policyNames, spec.Name)
+ }
+
+ var query string
+ var args []interface{}
+ var err error
+ if teamID == 0 {
+ query, args, err = sqlx.In("SELECT name, query, platforms FROM policies WHERE team_id IS NULL AND name IN (?)", policyNames)
+ } else {
+ query, args, err = sqlx.In(
+ "SELECT name, query, platforms FROM policies WHERE team_id = ? AND name IN (?)", &teamID, policyNames,
+ )
+ }
+ if err != nil {
+ return ctxerr.Wrap(ctx, err, "building query to get policies by name")
+ }
+ policies := make([]policyLite, 0, len(teamPolicySpecs))
+ err = sqlx.SelectContext(ctx, queryerContext, &policies, query, args...)
+ if err != nil {
+ return ctxerr.Wrap(ctx, err, "getting policies by name")
+ }
+ for _, p := range policies {
+ teamIDToPoliciesByName[teamID][p.Name] = p
+ }
+ }
+
return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error {
query := fmt.Sprintf(
`
@@ -592,7 +689,7 @@ func (ds *Datastore) ApplyPolicySpecs(ctx context.Context, authorID uint, specs
critical,
calendar_events_enabled,
checksum
- ) VALUES ( ?, ?, ?, ?, ?, (SELECT IFNULL(MIN(id), NULL) FROM teams WHERE name = ?), ?, ?, ?, %s)
+ ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, %s)
ON DUPLICATE KEY UPDATE
query = VALUES(query),
description = VALUES(description),
@@ -603,24 +700,45 @@ func (ds *Datastore) ApplyPolicySpecs(ctx context.Context, authorID uint, specs
calendar_events_enabled = VALUES(calendar_events_enabled)
`, policiesChecksumComputedColumn(),
)
- for _, spec := range specs {
-
- // We must normalize the name for full Unicode support (Unicode equivalence).
- spec.Name = norm.NFC.String(spec.Name)
- res, err := tx.ExecContext(ctx,
- query, spec.Name, spec.Query, spec.Description, authorID, spec.Resolution, spec.Team, spec.Platform, spec.Critical,
- spec.CalendarEventsEnabled,
- )
- if err != nil {
- return ctxerr.Wrap(ctx, err, "exec ApplyPolicySpecs insert")
+ for teamID, teamPolicySpecs := range teamIDToPolicies {
+ var teamIDPtr *uint
+ if teamID != 0 {
+ teamIDPtr = &teamID
}
+ for _, spec := range teamPolicySpecs {
- if insertOnDuplicateDidUpdate(res) {
- // when the upsert results in an UPDATE that *did* change some values,
- // it returns the updated ID as last inserted id.
- if lastID, _ := res.LastInsertId(); lastID > 0 {
- if err := cleanupPolicyMembershipOnPolicyUpdate(ctx, tx, uint(lastID), spec.Platform); err != nil {
- return err
+ res, err := tx.ExecContext(
+ ctx,
+ query, spec.Name, spec.Query, spec.Description, authorID, spec.Resolution, teamIDPtr, spec.Platform, spec.Critical,
+ spec.CalendarEventsEnabled,
+ )
+ if err != nil {
+ return ctxerr.Wrap(ctx, err, "exec ApplyPolicySpecs insert")
+ }
+
+ if insertOnDuplicateDidUpdate(res) {
+ // when the upsert results in an UPDATE that *did* change some values,
+ // it returns the updated ID as last inserted id.
+ if lastID, _ := res.LastInsertId(); lastID > 0 {
+ var (
+ shouldRemoveAllPolicyMemberships bool
+ removePolicyStats bool
+ )
+ // Figure out if the query or platform changed
+ if prev, ok := teamIDToPoliciesByName[teamID][spec.Name]; ok {
+ switch {
+ case prev.Query != spec.Query:
+ shouldRemoveAllPolicyMemberships = true
+ removePolicyStats = true
+ case prev.Platforms != spec.Platform:
+ removePolicyStats = true
+ }
+ }
+ if err = cleanupPolicy(
+ ctx, tx, uint(lastID), spec.Platform, shouldRemoveAllPolicyMemberships, removePolicyStats, ds.logger,
+ ); err != nil {
+ return err
+ }
}
}
}
@@ -739,7 +857,7 @@ func cleanupPolicyMembershipOnPolicyUpdate(ctx context.Context, db sqlx.ExecerCo
// cleanupPolicyMembership is similar to cleanupPolicyMembershipOnPolicyUpdate but without the platform constraints.
// Used when we want to remove all policy membership.
-func (ds *Datastore) cleanupPolicyMembershipForPolicy(ctx context.Context, policyID uint) error {
+func cleanupPolicyMembershipForPolicy(ctx context.Context, exec sqlx.ExecerContext, policyID uint) error {
// delete all policy memberships for the policy
delStmt := `
DELETE
@@ -754,21 +872,11 @@ func (ds *Datastore) cleanupPolicyMembershipForPolicy(ctx context.Context, polic
pm.policy_id = ?
`
- _, err := ds.writer(ctx).ExecContext(ctx, delStmt, policyID)
+ _, err := exec.ExecContext(ctx, delStmt, policyID)
if err != nil {
return ctxerr.Wrap(ctx, err, "cleanup policy membership")
}
- // delete all policy stats for the policy
- // wrapping in a retry to avoid deadlocks with the cleanups_then_aggregation cron job
- err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error {
- _, err := tx.ExecContext(ctx, `DELETE FROM policy_stats WHERE policy_id = ?`, policyID)
- return err
- })
- if err != nil {
- return ctxerr.Wrap(ctx, err, "cleanup policy stats")
- }
-
return nil
}
diff --git a/server/datastore/mysql/policies_test.go b/server/datastore/mysql/policies_test.go
index 045d850fa1..90d9015489 100644
--- a/server/datastore/mysql/policies_test.go
+++ b/server/datastore/mysql/policies_test.go
@@ -40,6 +40,7 @@ func TestPolicies(t *testing.T) {
{"PoliciesByID", testPoliciesByID},
{"TeamPolicyTransfer", testTeamPolicyTransfer},
{"ApplyPolicySpec", testApplyPolicySpec},
+ {"ApplyPolicySpecWithQueryPlatformChanges", testApplyPolicySpecWithQueryPlatformChanges},
{"Save", testPoliciesSave},
{"DelUser", testPoliciesDelUser},
{"FlippingPoliciesForHost", testFlippingPoliciesForHost},
@@ -1400,6 +1401,298 @@ func testApplyPolicySpec(t *testing.T, ds *Datastore) {
}))
}
+func testApplyPolicySpecWithQueryPlatformChanges(t *testing.T, ds *Datastore) {
+ ctx := context.Background()
+ unicode, _ := strconv.Unquote(`"\uAC00"`) // 가
+ unicodeEq, _ := strconv.Unquote(`"\u1100\u1161"`) // ᄀ + ᅡ
+
+ user1 := test.NewUser(t, ds, "User1", "user1@example.com", true)
+ team1, err := ds.NewTeam(ctx, &fleet.Team{Name: "team1" + unicode})
+ require.NoError(t, err)
+
+ globalNames := []string{"global query1" + unicode, "global query2" + unicode, "global query3" + unicode}
+ teamNames := []string{"team query1", "team query2", "team query3"}
+ require.NoError(
+ t, ds.ApplyPolicySpecs(
+ ctx, user1.ID, []*fleet.PolicySpec{
+ {
+ Name: globalNames[0],
+ Query: "select 1;",
+ Team: "",
+ Platform: "",
+ },
+ {
+ Name: globalNames[1],
+ Query: "select 2;",
+ Team: "",
+ Platform: "darwin",
+ },
+ {
+ Name: globalNames[2],
+ Query: "select 3;",
+ Team: "",
+ Platform: "darwin,linux",
+ },
+ {
+ Name: teamNames[0],
+ Query: "select 1;",
+ Team: "team1" + unicode,
+ Platform: "",
+ },
+ {
+ Name: teamNames[1],
+ Query: "select 2;",
+ Team: "team1" + unicode,
+ Platform: "darwin",
+ },
+ {
+ Name: teamNames[2],
+ Query: "select 3;",
+ Team: "team1" + unicodeEq,
+ Platform: "darwin,linux",
+ },
+ },
+ ),
+ )
+
+ // create hosts with different platforms, for that team
+ const hostWin, hostMac, hostDeb, hostLin = 0, 1, 2, 3
+ platforms := []string{"windows", "darwin", "debian", "linux"}
+ teamHosts := make([]*fleet.Host, len(platforms))
+ for i, pl := range platforms {
+ id := fmt.Sprintf("%s-%d", strings.ReplaceAll(t.Name(), "/", "_"), i)
+ h, err := ds.NewHost(
+ ctx, &fleet.Host{
+ OsqueryHostID: &id,
+ DetailUpdatedAt: time.Now(),
+ LabelUpdatedAt: time.Now(),
+ PolicyUpdatedAt: time.Now(),
+ SeenTime: time.Now(),
+ NodeKey: &id,
+ UUID: id,
+ Hostname: id,
+ Platform: pl,
+ TeamID: ptr.Uint(team1.ID),
+ },
+ )
+ require.NoError(t, err)
+ teamHosts[i] = h
+ }
+
+ // create hosts with different platforms, without team
+ globalHosts := make([]*fleet.Host, len(platforms))
+ for i, pl := range platforms {
+ id := fmt.Sprintf("g%s-%d", strings.ReplaceAll(t.Name(), "/", "_"), i)
+ h, err := ds.NewHost(
+ ctx, &fleet.Host{
+ OsqueryHostID: &id,
+ DetailUpdatedAt: time.Now(),
+ LabelUpdatedAt: time.Now(),
+ PolicyUpdatedAt: time.Now(),
+ SeenTime: time.Now(),
+ NodeKey: &id,
+ UUID: id,
+ Hostname: id,
+ Platform: pl,
+ },
+ )
+ require.NoError(t, err)
+ globalHosts[i] = h
+ }
+
+ // load the global policies
+ gPolicies, err := ds.ListGlobalPolicies(ctx, fleet.ListOptions{})
+ require.NoError(t, err)
+ require.Len(t, gPolicies, 3)
+ // load the team policies
+ tPolicies, _, err := ds.ListTeamPolicies(ctx, team1.ID, fleet.ListOptions{}, fleet.ListOptions{})
+ require.NoError(t, err)
+ require.Len(t, tPolicies, 3)
+
+ // index the policies by name for easier access in the rest of the test
+ polsByName := make(map[string]*fleet.Policy, len(gPolicies)+len(tPolicies))
+ globalPolsByName := make(map[string]*fleet.Policy, len(gPolicies))
+ for _, pol := range tPolicies {
+ polsByName[pol.Name] = pol
+ }
+ for _, pol := range gPolicies {
+ globalPolsByName[pol.Name] = pol
+ polsByName[pol.Name] = pol
+ }
+
+ // record some results for each policy
+ // Note: we are adding results to hosts that shouldn't have results, based on their platform.
+ for _, h := range teamHosts {
+ res := make(map[uint]*bool, len(polsByName))
+ for _, pol := range polsByName {
+ res[pol.ID] = ptr.Bool(false)
+ }
+ err = ds.RecordPolicyQueryExecutions(ctx, h, res, time.Now(), false)
+ require.NoError(t, err)
+ }
+ for _, h := range globalHosts {
+ res := make(map[uint]*bool, len(globalPolsByName))
+ for _, pol := range globalPolsByName {
+ res[pol.ID] = ptr.Bool(false)
+ }
+ err = ds.RecordPolicyQueryExecutions(ctx, h, res, time.Now(), false)
+ require.NoError(t, err)
+ }
+ err = ds.UpdateHostPolicyCounts(ctx)
+ require.NoError(t, err)
+
+ // Update host failure counts and ensure they are correct
+ teamHosts, err = ds.UpdatePolicyFailureCountsForHosts(ctx, teamHosts)
+ require.NoError(t, err)
+ assert.Equal(t, 6, teamHosts[hostWin].FailingPoliciesCount)
+ assert.Equal(t, 6, teamHosts[hostMac].FailingPoliciesCount)
+ assert.Equal(t, 6, teamHosts[hostDeb].FailingPoliciesCount)
+ assert.Equal(t, 6, teamHosts[hostLin].FailingPoliciesCount)
+ globalHosts, err = ds.UpdatePolicyFailureCountsForHosts(ctx, globalHosts)
+ require.NoError(t, err)
+ assert.Equal(t, 3, globalHosts[hostWin].FailingPoliciesCount)
+ assert.Equal(t, 3, globalHosts[hostMac].FailingPoliciesCount)
+ assert.Equal(t, 3, globalHosts[hostDeb].FailingPoliciesCount)
+ assert.Equal(t, 3, globalHosts[hostLin].FailingPoliciesCount)
+
+ // Ensure policy passing and failing counts are correct
+ gPolicies, err = ds.ListGlobalPolicies(ctx, fleet.ListOptions{})
+ require.NoError(t, err)
+ require.Len(t, gPolicies, 3)
+ tPolicies, _, err = ds.ListTeamPolicies(ctx, team1.ID, fleet.ListOptions{}, fleet.ListOptions{})
+ require.NoError(t, err)
+ require.Len(t, tPolicies, 3)
+
+ for _, pol := range gPolicies {
+ polsByName[pol.Name] = pol
+ }
+ for _, pol := range tPolicies {
+ polsByName[pol.Name] = pol
+ }
+ assert.Equal(t, uint(8), polsByName[globalNames[0]].FailingHostCount)
+ assert.Equal(t, uint(8), polsByName[globalNames[1]].FailingHostCount)
+ assert.Equal(t, uint(8), polsByName[globalNames[2]].FailingHostCount)
+ assert.Equal(t, uint(4), polsByName[teamNames[0]].FailingHostCount)
+ assert.Equal(t, uint(4), polsByName[teamNames[1]].FailingHostCount)
+ assert.Equal(t, uint(4), polsByName[teamNames[2]].FailingHostCount)
+
+ // Update policies
+ require.NoError(
+ t, ds.ApplyPolicySpecs(
+ ctx, user1.ID, []*fleet.PolicySpec{
+ {
+ Name: globalNames[0],
+ Query: "select 1;",
+ Team: "",
+ Platform: "",
+ Description: "updated", // update description
+ },
+ {
+ Name: globalNames[1],
+ Query: "select 2 updated;", // update query
+ Team: "",
+ Platform: "darwin",
+ },
+ {
+ Name: globalNames[2],
+ Query: "select 3;",
+ Team: "",
+ Platform: "darwin", // update platform
+ },
+ {
+ Name: "new global query",
+ Query: "select 4;",
+ Team: "",
+ Platform: "",
+ },
+ {
+ Name: teamNames[0],
+ Query: "select 1;",
+ Team: "team1" + unicode,
+ Platform: "linux", // update platform
+ },
+ {
+ Name: teamNames[1],
+ Query: "select 2;",
+ Team: "team1" + unicode,
+ Platform: "darwin",
+ CalendarEventsEnabled: true, // update calendar events
+ },
+ {
+ Name: teamNames[2],
+ Query: "select 3 updated;", // update query
+ Team: "team1" + unicodeEq,
+ Platform: "darwin,linux",
+ },
+ {
+ Name: "new team query",
+ Query: "select 4;",
+ Team: "team1" + unicode,
+ Platform: "",
+ },
+ },
+ ),
+ )
+
+ // Update host failure counts and ensure they are correct
+ teamHosts, err = ds.UpdatePolicyFailureCountsForHosts(ctx, teamHosts)
+ require.NoError(t, err)
+ assert.Equal(t, 1, teamHosts[hostWin].FailingPoliciesCount) // kept result from globalNames[0]
+ assert.Equal(t, 3, teamHosts[hostMac].FailingPoliciesCount)
+ assert.Equal(t, 2, teamHosts[hostDeb].FailingPoliciesCount)
+ assert.Equal(t, 2, teamHosts[hostLin].FailingPoliciesCount)
+ globalHosts, err = ds.UpdatePolicyFailureCountsForHosts(ctx, globalHosts)
+ require.NoError(t, err)
+ assert.Equal(t, 1, globalHosts[hostWin].FailingPoliciesCount)
+ assert.Equal(t, 2, globalHosts[hostMac].FailingPoliciesCount)
+ assert.Equal(t, 1, globalHosts[hostDeb].FailingPoliciesCount)
+ assert.Equal(t, 1, globalHosts[hostLin].FailingPoliciesCount)
+
+ // Ensure policy passing and failing counts are correct
+ gPolicies, err = ds.ListGlobalPolicies(ctx, fleet.ListOptions{})
+ require.NoError(t, err)
+ require.Len(t, gPolicies, 4)
+ tPolicies, _, err = ds.ListTeamPolicies(ctx, team1.ID, fleet.ListOptions{}, fleet.ListOptions{})
+ require.NoError(t, err)
+ require.Len(t, tPolicies, 4)
+
+ for _, pol := range gPolicies {
+ polsByName[pol.Name] = pol
+ }
+ for _, pol := range tPolicies {
+ polsByName[pol.Name] = pol
+ }
+ assert.Equal(t, uint(8), polsByName[globalNames[0]].FailingHostCount)
+ assert.Equal(t, uint(0), polsByName[globalNames[1]].FailingHostCount) // updated query
+ assert.Equal(t, uint(0), polsByName[globalNames[2]].FailingHostCount) // updated platform
+ assert.Equal(t, uint(0), polsByName[teamNames[0]].FailingHostCount) // updated platform
+ assert.Equal(t, uint(4), polsByName[teamNames[1]].FailingHostCount)
+ assert.Equal(t, uint(0), polsByName[teamNames[2]].FailingHostCount) // updated query
+
+ err = ds.UpdateHostPolicyCounts(ctx)
+ require.NoError(t, err)
+ gPolicies, err = ds.ListGlobalPolicies(ctx, fleet.ListOptions{})
+ require.NoError(t, err)
+ require.Len(t, gPolicies, 4)
+ tPolicies, _, err = ds.ListTeamPolicies(ctx, team1.ID, fleet.ListOptions{}, fleet.ListOptions{})
+ require.NoError(t, err)
+ require.Len(t, tPolicies, 4)
+
+ for _, pol := range gPolicies {
+ polsByName[pol.Name] = pol
+ }
+ for _, pol := range tPolicies {
+ polsByName[pol.Name] = pol
+ }
+ assert.Equal(t, uint(8), polsByName[globalNames[0]].FailingHostCount) // platform is "" -- no change
+ assert.Equal(t, uint(0), polsByName[globalNames[1]].FailingHostCount) // updated query
+ assert.Equal(t, uint(2), polsByName[globalNames[2]].FailingHostCount) // updated platform
+ assert.Equal(t, uint(2), polsByName[teamNames[0]].FailingHostCount) // updated platform
+ assert.Equal(t, uint(1), polsByName[teamNames[1]].FailingHostCount) // platform is "darwin" -- no change
+ assert.Equal(t, uint(0), polsByName[teamNames[2]].FailingHostCount) // updated query
+
+}
+
func testPoliciesSave(t *testing.T, ds *Datastore) {
user1 := test.NewUser(t, ds, "User1", "user1@example.com", true)
ctx := context.Background()
@@ -1412,7 +1705,8 @@ func testPoliciesSave(t *testing.T, ds *Datastore) {
Name: "non-existent query",
Query: "select 1;",
},
- }, false)
+ }, false, false,
+ )
require.Error(t, err)
var nfe *notFoundError
require.True(t, errors.As(err, &nfe))
@@ -1473,7 +1767,7 @@ func testPoliciesSave(t *testing.T, ds *Datastore) {
gp2 := *gp
gp2.Name = "global query updated"
gp2.Critical = true
- err = ds.SavePolicy(ctx, &gp2, false)
+ err = ds.SavePolicy(ctx, &gp2, false, false)
require.NoError(t, err)
gp, err = ds.Policy(ctx, gp.ID)
require.NoError(t, err)
@@ -1493,7 +1787,7 @@ func testPoliciesSave(t *testing.T, ds *Datastore) {
tp2.Resolution = ptr.String("team1 query resolution updated")
tp2.Critical = false
tp2.CalendarEventsEnabled = false
- err = ds.SavePolicy(ctx, &tp2, true)
+ err = ds.SavePolicy(ctx, &tp2, true, true)
require.NoError(t, err)
tp1, err = ds.Policy(ctx, tp1.ID)
tp2.UpdateCreateTimestamps = tp1.UpdateCreateTimestamps
@@ -1586,7 +1880,7 @@ func testCachedPolicyCountDeletesOnPolicyChange(t *testing.T, ds *Datastore) {
assert.Equal(t, uint(1), inheritedPolicies[0].PassingHostCount)
// Update the global policy sql to trigger a cache invalidation
- err = ds.SavePolicy(ctx, globalPolicy, true)
+ err = ds.SavePolicy(ctx, globalPolicy, true, true)
require.NoError(t, err)
globalPolicy, err = ds.Policy(ctx, globalPolicy.ID)
@@ -1599,8 +1893,8 @@ func testCachedPolicyCountDeletesOnPolicyChange(t *testing.T, ds *Datastore) {
assert.Equal(t, uint(1), teamPolicies[0].PassingHostCount)
assert.Equal(t, uint(0), inheritedPolicies[0].PassingHostCount)
- // Update the team policy sql to trigger a cache invalidation
- err = ds.SavePolicy(ctx, teamPolicy, true)
+ // Update the team policy platform to trigger a cache invalidation
+ err = ds.SavePolicy(ctx, teamPolicy, false, true)
require.NoError(t, err)
teamPolicies, inheritedPolicies, err = ds.ListTeamPolicies(ctx, team1.ID, fleet.ListOptions{}, fleet.ListOptions{})
@@ -1921,9 +2215,9 @@ func testPolicyPlatformUpdate(t *testing.T, ds *Datastore) {
}
// updating without change works fine
- err = ds.SavePolicy(ctx, polsByName["g1"], false)
+ err = ds.SavePolicy(ctx, polsByName["g1"], false, false)
require.NoError(t, err)
- err = ds.SavePolicy(ctx, polsByName["t2"], false)
+ err = ds.SavePolicy(ctx, polsByName["t2"], false, false)
require.NoError(t, err)
// apply specs that result in an update (without change) works fine
err = ds.ApplyPolicySpecs(ctx, user.ID, []*fleet.PolicySpec{
@@ -1975,7 +2269,7 @@ func testPolicyPlatformUpdate(t *testing.T, ds *Datastore) {
g1 := polsByName["g1"]
g1.Platform = "linux"
polsByName["g1"] = g1
- err = ds.SavePolicy(ctx, g1, false)
+ err = ds.SavePolicy(ctx, g1, false, false)
require.NoError(t, err)
wantHostsByPol["g1"] = []uint{globalHosts[hostDeb].ID, globalHosts[hostLin].ID}
assertPolicyMembership(t, ds, polsByName, wantHostsByPol)
@@ -1984,7 +2278,7 @@ func testPolicyPlatformUpdate(t *testing.T, ds *Datastore) {
t1 := polsByName["t1"]
t1.Platform = "windows,darwin"
polsByName["t1"] = t1
- err = ds.SavePolicy(ctx, t1, false)
+ err = ds.SavePolicy(ctx, t1, false, false)
require.NoError(t, err)
wantHostsByPol["t1"] = []uint{teamHosts[hostWin].ID, teamHosts[hostMac].ID}
assertPolicyMembership(t, ds, polsByName, wantHostsByPol)
@@ -2723,7 +3017,7 @@ func testPoliciesNameUnicode(t *testing.T, ds *Datastore) {
policyEmoji, err := ds.NewGlobalPolicy(context.Background(), nil, fleet.PolicyPayload{Name: "💻"})
require.NoError(t, err)
err = ds.SavePolicy(
- context.Background(), &fleet.Policy{PolicyData: fleet.PolicyData{ID: policyEmoji.ID, Name: equivalentNames[1]}}, false,
+ context.Background(), &fleet.Policy{PolicyData: fleet.PolicyData{ID: policyEmoji.ID, Name: equivalentNames[1]}}, false, false,
)
assert.True(t, isDuplicate(err), err)
@@ -3270,10 +3564,10 @@ func testGetTeamHostsPolicyMemberships(t *testing.T, ds *Datastore) {
//
team2Policy1.Platform = "darwin"
- err = ds.SavePolicy(ctx, team1Policy1, false)
+ err = ds.SavePolicy(ctx, team1Policy1, false, true)
require.NoError(t, err)
team1Policy1.Platform = "darwin"
- err = ds.SavePolicy(ctx, team2Policy1, false)
+ err = ds.SavePolicy(ctx, team2Policy1, false, true)
require.NoError(t, err)
//
diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql
index 783c8be08e..861cc47d6e 100644
--- a/server/datastore/mysql/schema.sql
+++ b/server/datastore/mysql/schema.sql
@@ -41,7 +41,7 @@ CREATE TABLE `app_config_json` (
UNIQUE KEY `id` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
-INSERT INTO `app_config_json` VALUES (1,'{\"mdm\": {\"macos_setup\": {\"bootstrap_package\": null, \"macos_setup_assistant\": null, \"enable_end_user_authentication\": false, \"enable_release_device_manually\": false}, \"macos_updates\": {\"deadline\": null, \"minimum_version\": null}, \"macos_settings\": {\"custom_settings\": null}, \"macos_migration\": {\"mode\": \"\", \"enable\": false, \"webhook_url\": \"\"}, \"windows_updates\": {\"deadline_days\": null, \"grace_period_days\": null}, \"windows_settings\": {\"custom_settings\": null}, \"apple_bm_default_team\": \"\", \"apple_bm_terms_expired\": false, \"enable_disk_encryption\": false, \"enabled_and_configured\": false, \"end_user_authentication\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"issuer_uri\": \"\", \"metadata_url\": \"\"}, \"windows_enabled_and_configured\": false, \"apple_bm_enabled_and_configured\": false}, \"scripts\": null, \"features\": {\"enable_host_users\": true, \"enable_software_inventory\": false}, \"org_info\": {\"org_name\": \"\", \"contact_url\": \"\", \"org_logo_url\": \"\", \"org_logo_url_light_background\": \"\"}, \"integrations\": {\"jira\": null, \"zendesk\": null, \"google_calendar\": null}, \"sso_settings\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"enable_sso\": false, \"issuer_uri\": \"\", \"metadata_url\": \"\", \"idp_image_url\": \"\", \"enable_jit_role_sync\": false, \"enable_sso_idp_login\": false, \"enable_jit_provisioning\": false}, \"agent_options\": {\"config\": {\"options\": {\"logger_plugin\": \"tls\", \"pack_delimiter\": \"/\", \"logger_tls_period\": 10, \"distributed_plugin\": \"tls\", \"disable_distributed\": false, \"logger_tls_endpoint\": \"/api/osquery/log\", \"distributed_interval\": 10, \"distributed_tls_max_attempts\": 3}, \"decorators\": {\"load\": [\"SELECT uuid AS host_uuid FROM system_info;\", \"SELECT hostname AS hostname FROM system_info;\"]}}, \"overrides\": {}}, \"fleet_desktop\": {\"transparency_url\": \"\"}, \"smtp_settings\": {\"port\": 587, \"domain\": \"\", \"server\": \"\", \"password\": \"\", \"user_name\": \"\", \"configured\": false, \"enable_smtp\": false, \"enable_ssl_tls\": true, \"sender_address\": \"\", \"enable_start_tls\": true, \"verify_ssl_certs\": true, \"authentication_type\": \"0\", \"authentication_method\": \"0\"}, \"server_settings\": {\"server_url\": \"\", \"enable_analytics\": false, \"scripts_disabled\": false, \"deferred_save_host\": false, \"live_query_disabled\": false, \"query_reports_disabled\": false}, \"webhook_settings\": {\"interval\": \"0s\", \"host_status_webhook\": {\"days_count\": 0, \"destination_url\": \"\", \"host_percentage\": 0, \"enable_host_status_webhook\": false}, \"vulnerabilities_webhook\": {\"destination_url\": \"\", \"host_batch_size\": 0, \"enable_vulnerabilities_webhook\": false}, \"failing_policies_webhook\": {\"policy_ids\": null, \"destination_url\": \"\", \"host_batch_size\": 0, \"enable_failing_policies_webhook\": false}}, \"host_expiry_settings\": {\"host_expiry_window\": 0, \"host_expiry_enabled\": false}, \"vulnerability_settings\": {\"databases_path\": \"\"}}','2020-01-01 01:01:01','2020-01-01 01:01:01');
+INSERT INTO `app_config_json` VALUES (1,'{\"mdm\": {\"macos_setup\": {\"bootstrap_package\": null, \"macos_setup_assistant\": null, \"enable_end_user_authentication\": false, \"enable_release_device_manually\": false}, \"macos_updates\": {\"deadline\": null, \"minimum_version\": null}, \"macos_settings\": {\"custom_settings\": null}, \"macos_migration\": {\"mode\": \"\", \"enable\": false, \"webhook_url\": \"\"}, \"windows_updates\": {\"deadline_days\": null, \"grace_period_days\": null}, \"windows_settings\": {\"custom_settings\": null}, \"apple_bm_default_team\": \"\", \"apple_bm_terms_expired\": false, \"enable_disk_encryption\": false, \"enabled_and_configured\": false, \"end_user_authentication\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"issuer_uri\": \"\", \"metadata_url\": \"\"}, \"windows_enabled_and_configured\": false, \"apple_bm_enabled_and_configured\": false}, \"scripts\": null, \"features\": {\"enable_host_users\": true, \"enable_software_inventory\": false}, \"org_info\": {\"org_name\": \"\", \"contact_url\": \"\", \"org_logo_url\": \"\", \"org_logo_url_light_background\": \"\"}, \"integrations\": {\"jira\": null, \"zendesk\": null, \"google_calendar\": null}, \"sso_settings\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"enable_sso\": false, \"issuer_uri\": \"\", \"metadata_url\": \"\", \"idp_image_url\": \"\", \"enable_jit_role_sync\": false, \"enable_sso_idp_login\": false, \"enable_jit_provisioning\": false}, \"agent_options\": {\"config\": {\"options\": {\"logger_plugin\": \"tls\", \"pack_delimiter\": \"/\", \"logger_tls_period\": 10, \"distributed_plugin\": \"tls\", \"disable_distributed\": false, \"logger_tls_endpoint\": \"/api/osquery/log\", \"distributed_interval\": 10, \"distributed_tls_max_attempts\": 3}, \"decorators\": {\"load\": [\"SELECT uuid AS host_uuid FROM system_info;\", \"SELECT hostname AS hostname FROM system_info;\"]}}, \"overrides\": {}}, \"fleet_desktop\": {\"transparency_url\": \"\"}, \"smtp_settings\": {\"port\": 587, \"domain\": \"\", \"server\": \"\", \"password\": \"\", \"user_name\": \"\", \"configured\": false, \"enable_smtp\": false, \"enable_ssl_tls\": true, \"sender_address\": \"\", \"enable_start_tls\": true, \"verify_ssl_certs\": true, \"authentication_type\": \"0\", \"authentication_method\": \"0\"}, \"server_settings\": {\"server_url\": \"\", \"enable_analytics\": false, \"scripts_disabled\": false, \"deferred_save_host\": false, \"live_query_disabled\": false, \"query_reports_disabled\": false}, \"webhook_settings\": {\"interval\": \"0s\", \"host_status_webhook\": {\"days_count\": 0, \"destination_url\": \"\", \"host_percentage\": 0, \"enable_host_status_webhook\": false}, \"vulnerabilities_webhook\": {\"destination_url\": \"\", \"host_batch_size\": 0, \"enable_vulnerabilities_webhook\": false}, \"failing_policies_webhook\": {\"policy_ids\": null, \"destination_url\": \"\", \"host_batch_size\": 0, \"enable_failing_policies_webhook\": false}}, \"host_expiry_settings\": {\"host_expiry_window\": 0, \"host_expiry_enabled\": false}, \"vulnerability_settings\": {\"databases_path\": \"\"}, \"activity_expiry_settings\": {\"activity_expiry_window\": 0, \"activity_expiry_enabled\": false}}','2020-01-01 01:01:01','2020-01-01 01:01:01');
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `calendar_events` (
diff --git a/server/fleet/app.go b/server/fleet/app.go
index ac056347f0..8f4525a595 100644
--- a/server/fleet/app.go
+++ b/server/fleet/app.go
@@ -434,8 +434,9 @@ type AppConfig struct {
// SMTPSettings holds the SMTP integration settings.
//
// This field is a pointer to avoid returning this information to non-global-admins.
- SMTPSettings *SMTPSettings `json:"smtp_settings,omitempty"`
- HostExpirySettings HostExpirySettings `json:"host_expiry_settings"`
+ SMTPSettings *SMTPSettings `json:"smtp_settings,omitempty"`
+ HostExpirySettings HostExpirySettings `json:"host_expiry_settings"`
+ ActivityExpirySettings ActivityExpirySettings `json:"activity_expiry_settings"`
// Features allows to globally enable or disable features
Features Features `json:"features"`
DeprecatedHostSettings *Features `json:"host_settings,omitempty"`
@@ -888,6 +889,12 @@ type HostExpirySettings struct {
HostExpiryWindow int `json:"host_expiry_window"`
}
+// ActivityExpirySettings contains settings pertaining to automatic activities cleanup.
+type ActivityExpirySettings struct {
+ ActivityExpiryEnabled bool `json:"activity_expiry_enabled"`
+ ActivityExpiryWindow int `json:"activity_expiry_window"`
+}
+
type Features struct {
EnableHostUsers bool `json:"enable_host_users"`
EnableSoftwareInventory bool `json:"enable_software_inventory"`
diff --git a/server/fleet/apple_mdm.go b/server/fleet/apple_mdm.go
index 71de52073a..04f5742e61 100644
--- a/server/fleet/apple_mdm.go
+++ b/server/fleet/apple_mdm.go
@@ -161,15 +161,6 @@ type EnrolledAPIResult struct {
// EnrolledAPIResults is a map of enrollments to a per-enrollment API result.
type EnrolledAPIResults map[string]*EnrolledAPIResult
-// MDMAppleHostDetails represents the device identifiers used to ingest an MDM device as a Fleet
-// host pending enrollment.
-// See also https://developer.apple.com/documentation/devicemanagement/authenticaterequest.
-type MDMAppleHostDetails struct {
- SerialNumber string
- UDID string
- Model string
-}
-
type MDMAppleCommandTimeoutError struct{}
func (e MDMAppleCommandTimeoutError) Error() string {
diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go
index 36c6ff1dd1..d5d1a26d6a 100644
--- a/server/fleet/datastore.go
+++ b/server/fleet/datastore.go
@@ -570,7 +570,10 @@ type Datastore interface {
// upgraded from a prior version).
CleanupHostOperatingSystems(ctx context.Context) error
- UpdateHostTablesOnMDMUnenroll(ctx context.Context, uuid string) error
+ // MDMTurnOff updates Fleet host information related to MDM when a
+ // host turns off MDM. Anything related to the protocol itself is
+ // managed separately.
+ MDMTurnOff(ctx context.Context, uuid string) error
///////////////////////////////////////////////////////////////////////////////
// ActivitiesStore
@@ -604,7 +607,7 @@ type Datastore interface {
// SavePolicy updates some fields of the given policy on the datastore.
//
// It is also used to update team policies.
- SavePolicy(ctx context.Context, p *Policy, shouldRemoveAllPolicyMemberships bool) error
+ SavePolicy(ctx context.Context, p *Policy, shouldRemoveAllPolicyMemberships bool, removePolicyStats bool) error
ListGlobalPolicies(ctx context.Context, opts ListOptions) ([]*Policy, error)
PoliciesByID(ctx context.Context, ids []uint) (map[uint]*Policy, error)
@@ -1042,16 +1045,16 @@ type Datastore interface {
// joined (nil for no team), and an error.
IngestMDMAppleDevicesFromDEPSync(ctx context.Context, devices []godep.Device) (int64, *uint, error)
- // IngestMDMAppleDeviceFromCheckin creates a new Fleet host record for an MDM-enrolled device that is
- // not already enrolled in Fleet.
- IngestMDMAppleDeviceFromCheckin(ctx context.Context, mdmHost MDMAppleHostDetails) error
+ // MDMAppleUpsertHost creates or matches a Fleet host record for an
+ // MDM-enrolled device.
+ MDMAppleUpsertHost(ctx context.Context, mdmHost *Host) error
// RestoreMDMApplePendingDEPHost restores a host that was previously deleted from Fleet.
RestoreMDMApplePendingDEPHost(ctx context.Context, host *Host) error
- // ResetMDMAppleEnrollment resets all tables with enrollment-related
+ // MDMResetEnrollment resets all tables with enrollment-related
// information if a matching row for the host exists.
- ResetMDMAppleEnrollment(ctx context.Context, hostUUID string) error
+ MDMResetEnrollment(ctx context.Context, hostUUID string) error
// ListMDMAppleDEPSerialsInTeam returns a list of serial numbers of hosts
// that are enrolled or pending enrollment in Fleet's MDM via DEP for the
@@ -1432,6 +1435,11 @@ type Datastore interface {
// CleanupUnusedScriptContents will remove script contents that have no references to them from
// the scripts or host_script_results tables.
CleanupUnusedScriptContents(ctx context.Context) error
+ // CleanupActivitiesAndAssociatedData will cleanup (up to maxCount) activities and their associated data
+ // that are older than the given expiration window.
+ //
+ // The argument maxCount is used to not lock the database for long periods of time.
+ CleanupActivitiesAndAssociatedData(ctx context.Context, maxCount int, expiryWindowDays int) error
// WipeHostViaScript sends a script to wipe a host and updates the
// states in host_mdm_actions.
WipeHostViaScript(ctx context.Context, request *HostScriptRequestPayload, hostFleetPlatform string) error
diff --git a/server/fleet/hosts.go b/server/fleet/hosts.go
index ae0124c0d8..0ed16f60c3 100644
--- a/server/fleet/hosts.go
+++ b/server/fleet/hosts.go
@@ -361,7 +361,9 @@ type Host struct {
// HostOrbitInfo maps to the host_orbit_info table in the database, which maps to the orbit_info agent table.
type HostOrbitInfo struct {
- ScriptsEnabled *bool `json:"scripts_enabled" db:"scripts_enabled"`
+ Version string `json:"version" db:"version"`
+ DesktopVersion *string `json:"desktop_version" db:"desktop_version"`
+ ScriptsEnabled *bool `json:"scripts_enabled" db:"scripts_enabled"`
}
// HostHealth contains a subset of Host data that indicates how healthy a Host is. For fields with
diff --git a/server/mdm/apple/mobileconfig/mobileconfig.go b/server/mdm/apple/mobileconfig/mobileconfig.go
index ac7b11f89a..8fecaed417 100644
--- a/server/mdm/apple/mobileconfig/mobileconfig.go
+++ b/server/mdm/apple/mobileconfig/mobileconfig.go
@@ -8,8 +8,11 @@ import (
"github.com/fleetdm/fleet/v4/server/config"
"github.com/fleetdm/fleet/v4/server/mdm"
+
+ // we are using this package as we were having issues with pasrsing signed apple
+ // mobileconfig profiles with the pcks7 package we were using before.
+ cms "github.com/github/smimesign/ietf-cms"
"github.com/micromdm/micromdm/pkg/crypto/profileutil"
- "go.mozilla.org/pkcs7"
"howett.net/plist"
)
@@ -81,6 +84,24 @@ type Parsed struct {
PayloadType string
}
+func (mc Mobileconfig) isSignedProfile() bool {
+ return !bytes.HasPrefix(bytes.TrimSpace(mc), []byte(" 0 {
- err := svc.checkWriteForHostIDs(ctx, ids)
+ if err := svc.checkWriteForHostIDs(ctx, ids); err != nil {
+ return err
+ }
+
+ hosts, err := svc.ds.ListHostsLiteByIDs(ctx, ids)
if err != nil {
return err
}
- return svc.ds.DeleteHosts(ctx, ids)
+
+ return doDelete(ids, hosts)
}
if opts == nil {
opts = &fleet.HostListOptions{}
}
opts.DisableFailingPolicies = true // don't check policies for hosts that are about to be deleted
- hostIDs, _, err := svc.hostIDsAndNamesFromFilters(ctx, *opts, lid)
+ hostIDs, _, hosts, err := svc.hostIDsAndNamesFromFilters(ctx, *opts, lid)
if err != nil {
return err
}
@@ -309,7 +335,8 @@ func (svc *Service) DeleteHosts(ctx context.Context, ids []uint, filter *map[str
if err != nil {
return err
}
- return svc.ds.DeleteHosts(ctx, hostIDs)
+
+ return doDelete(hostIDs, hosts)
}
/////////////////////////////////////////////////////////////////////////////////
@@ -719,8 +746,15 @@ func (svc *Service) DeleteHost(ctx context.Context, id uint) error {
return ctxerr.Wrap(ctx, err, "delete host")
}
- if host.Platform == "darwin" {
- return svc.maybeRestorePendingDEPHost(ctx, host)
+ if host.Platform == "windows" || host.Platform == "darwin" {
+ mdmLifecycle := mdmlifecycle.New(svc.ds, svc.logger)
+ err = mdmLifecycle.Do(ctx, mdmlifecycle.HostOptions{
+ Action: mdmlifecycle.HostActionDelete,
+ Platform: host.Platform,
+ UUID: host.UUID,
+ Host: host,
+ })
+ return ctxerr.Wrap(ctx, err, "performing MDM actions after delete")
}
return nil
@@ -888,7 +922,7 @@ func (svc *Service) AddHostsToTeamByFilter(ctx context.Context, teamID *uint, fi
return &fleet.BadRequestError{Message: "filters must be specified"}
}
- hostIDs, hostNames, err := svc.hostIDsAndNamesFromFilters(ctx, *opt, lid)
+ hostIDs, hostNames, _, err := svc.hostIDsAndNamesFromFilters(ctx, *opt, lid)
if err != nil {
return err
}
@@ -1241,10 +1275,10 @@ func (svc *Service) GetHostQueryReportResults(ctx context.Context, hostID uint,
return result, lastFetched, nil
}
-func (svc *Service) hostIDsAndNamesFromFilters(ctx context.Context, opt fleet.HostListOptions, lid *uint) ([]uint, []string, error) {
+func (svc *Service) hostIDsAndNamesFromFilters(ctx context.Context, opt fleet.HostListOptions, lid *uint) ([]uint, []string, []*fleet.Host, error) {
filter, err := processHostFilters(ctx, opt, lid)
if err != nil {
- return nil, nil, err
+ return nil, nil, nil, err
}
// Load hosts, either from label if provided or from all hosts.
@@ -1256,11 +1290,11 @@ func (svc *Service) hostIDsAndNamesFromFilters(ctx context.Context, opt fleet.Ho
hosts, err = svc.ds.ListHosts(ctx, filter, opt)
}
if err != nil {
- return nil, nil, err
+ return nil, nil, nil, err
}
if len(hosts) == 0 {
- return nil, nil, nil
+ return nil, nil, nil, nil
}
hostIDs := make([]uint, 0, len(hosts))
@@ -1269,7 +1303,7 @@ func (svc *Service) hostIDsAndNamesFromFilters(ctx context.Context, opt fleet.Ho
hostIDs = append(hostIDs, h.ID)
hostNames = append(hostNames, h.DisplayName())
}
- return hostIDs, hostNames, nil
+ return hostIDs, hostNames, hosts, nil
}
func processHostFilters(ctx context.Context, opt fleet.HostListOptions, lid *uint) (fleet.TeamFilter, error) {
diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go
index 0d5f371e44..35576136ed 100644
--- a/server/service/integration_core_test.go
+++ b/server/service/integration_core_test.go
@@ -2428,6 +2428,65 @@ func (s *integrationTestSuite) TestGlobalPoliciesProprietary() {
assert.Equal(t, uint(0), policiesResponse.Policies[0].FailingHostCount)
assert.Equal(t, uint(0), policiesResponse.Policies[0].PassingHostCount)
+ // Record query executions
+ require.NoError(
+ t, s.ds.RecordPolicyQueryExecutions(
+ context.Background(), h1.Host, map[uint]*bool{policiesResponse.Policies[0].ID: ptr.Bool(true)}, time.Now(), false,
+ ),
+ )
+ require.NoError(
+ t, s.ds.RecordPolicyQueryExecutions(
+ context.Background(), h2.Host, map[uint]*bool{policiesResponse.Policies[0].ID: nil}, time.Now(), false,
+ ),
+ )
+ // Update policy stats
+ require.NoError(t, s.ds.UpdateHostPolicyCounts(context.Background()))
+
+ // Fetch policy to make sure stats are updated
+ s.DoJSON("GET", "/api/latest/fleet/policies", nil, http.StatusOK, &policiesResponse)
+ require.Len(t, policiesResponse.Policies, 1)
+ assert.Equal(t, uint(0), policiesResponse.Policies[0].FailingHostCount)
+ assert.Equal(t, uint(1), policiesResponse.Policies[0].PassingHostCount)
+
+ listHostsURL = fmt.Sprintf("/api/latest/fleet/hosts?policy_id=%d&policy_response=passing", policiesResponse.Policies[0].ID)
+ listHostsResp = listHostsResponse{}
+ s.DoJSON("GET", listHostsURL, nil, http.StatusOK, &listHostsResp)
+ require.Len(t, listHostsResp.Hosts, 1)
+
+ // Modify the platform for the policy, which should clear the policy stats
+ mgpParams = modifyGlobalPolicyRequest{
+ ModifyPolicyPayload: fleet.ModifyPolicyPayload{
+ Platform: ptr.String("linux"),
+ },
+ }
+ mgpResp = modifyGlobalPolicyResponse{}
+ s.DoJSON("PATCH", fmt.Sprintf("/api/latest/fleet/policies/%d", gpResp.Policy.ID), mgpParams, http.StatusOK, &mgpResp)
+ require.NotNil(t, gpResp.Policy)
+ assert.Equal(t, "TestQuery4", mgpResp.Policy.Name)
+ assert.Equal(t, "select * from users;", mgpResp.Policy.Query)
+ assert.Equal(t, "Some description updated", mgpResp.Policy.Description)
+ require.NotNil(t, mgpResp.Policy.Resolution)
+ assert.Equal(t, "some global resolution updated", *mgpResp.Policy.Resolution)
+ assert.Equal(t, "linux", mgpResp.Policy.Platform)
+ assert.Equal(t, uint(0), mgpResp.Policy.FailingHostCount)
+ assert.Equal(t, uint(0), mgpResp.Policy.PassingHostCount)
+
+ // Fetch policy to make sure stats are updated
+ s.DoJSON("GET", "/api/latest/fleet/policies", nil, http.StatusOK, &policiesResponse)
+ require.Len(t, policiesResponse.Policies, 1)
+ assert.Equal(t, uint(0), policiesResponse.Policies[0].FailingHostCount)
+ assert.Equal(t, uint(0), policiesResponse.Policies[0].PassingHostCount)
+
+ listHostsURL = fmt.Sprintf("/api/latest/fleet/hosts?policy_id=%d&policy_response=passing", policiesResponse.Policies[0].ID)
+ listHostsResp = listHostsResponse{}
+ s.DoJSON("GET", listHostsURL, nil, http.StatusOK, &listHostsResp)
+ require.Len(t, listHostsResp.Hosts, 0)
+
+ listHostsURL = fmt.Sprintf("/api/latest/fleet/hosts?policy_id=%d&policy_response=failing", policiesResponse.Policies[0].ID)
+ listHostsResp = listHostsResponse{}
+ s.DoJSON("GET", listHostsURL, nil, http.StatusOK, &listHostsResp)
+ require.Len(t, listHostsResp.Hosts, 0)
+
deletePolicyParams := deleteGlobalPoliciesRequest{IDs: []uint{policiesResponse.Policies[0].ID}}
deletePolicyResp := deleteGlobalPoliciesResponse{}
s.DoJSON("POST", "/api/latest/fleet/policies/delete", deletePolicyParams, http.StatusOK, &deletePolicyResp)
@@ -6115,6 +6174,8 @@ func (s *integrationTestSuite) TestAppConfig() {
assert.Equal(t, "free", acResp.License.Tier)
assert.Equal(t, "FleetTest", acResp.OrgInfo.OrgName) // set in SetupSuite
assert.False(t, acResp.MDM.AppleBMTermsExpired)
+ assert.False(t, acResp.ActivityExpirySettings.ActivityExpiryEnabled)
+ assert.Zero(t, acResp.ActivityExpirySettings.ActivityExpiryWindow)
// set the apple BM terms expired flag, and the enabled and configured flags,
// we'll check again at the end of this test to make sure they weren't
@@ -6157,6 +6218,30 @@ func (s *integrationTestSuite) TestAppConfig() {
s.DoJSON("GET", "/api/latest/fleet/config", nil, http.StatusOK, &acResp)
require.Contains(t, string(*acResp.AgentOptions), `"logger_plugin": "tls"`) // default agent options has this setting
+ // Invalid activity expiry window.
+ acResp = appConfigResponse{}
+ s.DoJSON("PATCH", "/api/latest/fleet/config", json.RawMessage(`{
+ "activity_expiry_settings": {
+ "activity_expiry_enabled": true,
+ "activity_expiry_window": -1
+ }
+ }`), http.StatusUnprocessableEntity, &acResp)
+ s.DoJSON("GET", "/api/latest/fleet/config", nil, http.StatusOK, &acResp)
+ require.False(t, acResp.ActivityExpirySettings.ActivityExpiryEnabled)
+ require.Zero(t, acResp.ActivityExpirySettings.ActivityExpiryWindow)
+
+ // Valid activity expiry window.
+ acResp = appConfigResponse{}
+ s.DoJSON("PATCH", "/api/latest/fleet/config", json.RawMessage(`{
+ "activity_expiry_settings": {
+ "activity_expiry_enabled": true,
+ "activity_expiry_window": 42
+ }
+ }`), http.StatusOK, &acResp)
+ s.DoJSON("GET", "/api/latest/fleet/config", nil, http.StatusOK, &acResp)
+ require.True(t, acResp.ActivityExpirySettings.ActivityExpiryEnabled)
+ require.Equal(t, 42, acResp.ActivityExpirySettings.ActivityExpiryWindow)
+
// test a change that does clear the agent options (the field is provided but empty).
s.DoJSON("PATCH", "/api/latest/fleet/config", json.RawMessage(`{
"agent_options": {}
@@ -8863,6 +8948,20 @@ type validationErrResp struct {
} `json:"errors"`
}
+func setOrbitEnrollment(t *testing.T, h *fleet.Host, ds fleet.Datastore) string {
+ orbitKey := uuid.New().String()
+ _, err := ds.EnrollOrbit(context.Background(), false, fleet.OrbitHostInfo{
+ HardwareUUID: *h.OsqueryHostID,
+ HardwareSerial: h.HardwareSerial,
+ }, orbitKey, nil)
+ require.NoError(t, err)
+ err = ds.SetOrUpdateHostOrbitInfo(
+ context.Background(), h.ID, "1.22.0", sql.NullString{String: "42", Valid: true}, sql.NullBool{Bool: true, Valid: true},
+ )
+ require.NoError(t, err)
+ return orbitKey
+}
+
func createOrbitEnrolledHost(t *testing.T, os, suffix string, ds fleet.Datastore) *fleet.Host {
name := t.Name() + suffix
h, err := ds.NewHost(context.Background(), &fleet.Host{
@@ -8878,16 +8977,8 @@ func createOrbitEnrolledHost(t *testing.T, os, suffix string, ds fleet.Datastore
Platform: os,
})
require.NoError(t, err)
- orbitKey := uuid.New().String()
- _, err = ds.EnrollOrbit(context.Background(), false, fleet.OrbitHostInfo{
- HardwareUUID: *h.OsqueryHostID,
- HardwareSerial: h.HardwareSerial,
- }, orbitKey, nil)
- require.NoError(t, err)
- err = ds.SetOrUpdateHostOrbitInfo(
- context.Background(), h.ID, "1.22.0", sql.NullString{String: "42", Valid: true}, sql.NullBool{Bool: true, Valid: true},
- )
- require.NoError(t, err)
+
+ orbitKey := setOrbitEnrollment(t, h, ds)
h.OrbitNodeKey = &orbitKey
return h
}
diff --git a/server/service/integration_mdm_ddm_test.go b/server/service/integration_mdm_ddm_test.go
index 869277f246..4034e5540a 100644
--- a/server/service/integration_mdm_ddm_test.go
+++ b/server/service/integration_mdm_ddm_test.go
@@ -971,6 +971,9 @@ func (s *integrationMDMTestSuite) TestDDMTransactionRecording() {
// a second device requests tokens
_, mdmDeviceTwo := createHostThenEnrollMDM(s.ds, s.server.URL, t)
+ err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, s.logger)
+ require.NoError(t, err)
+
_, err = mdmDeviceTwo.DeclarativeManagement("tokens")
require.NoError(t, err)
verifyTransactionRecord(record{
diff --git a/server/service/integration_mdm_lifecycle_test.go b/server/service/integration_mdm_lifecycle_test.go
new file mode 100644
index 0000000000..bf660ec620
--- /dev/null
+++ b/server/service/integration_mdm_lifecycle_test.go
@@ -0,0 +1,762 @@
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "encoding/xml"
+ "fmt"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/fleetdm/fleet/v4/pkg/mdm/mdmtest"
+ "github.com/fleetdm/fleet/v4/server/config"
+ "github.com/fleetdm/fleet/v4/server/datastore/mysql"
+ "github.com/fleetdm/fleet/v4/server/fleet"
+ apple_mdm "github.com/fleetdm/fleet/v4/server/mdm/apple"
+ "github.com/fleetdm/fleet/v4/server/mdm/microsoft/syncml"
+ "github.com/fleetdm/fleet/v4/server/mdm/nanodep/godep"
+ "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/mdm"
+ "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/push"
+ "github.com/fleetdm/fleet/v4/server/ptr"
+ kitlog "github.com/go-kit/log"
+ "github.com/google/uuid"
+ "github.com/groob/plist"
+ "github.com/jmoiron/sqlx"
+ micromdm "github.com/micromdm/micromdm/mdm/mdm"
+ "github.com/stretchr/testify/require"
+ "go.mozilla.org/pkcs7"
+)
+
+// NOTE: the mantra for lifecycle events is:
+// - Noah: When MDM is turned on, install fleetd, bootstrap package (if DEP),
+// and profiles. Don't clear host vitals (everything you see on the Host
+// details page)
+// - Noah: On re-enrollment, don't clear host vitals.
+// - Noah: On lock and wipe, don't clear host vitals.
+// - Noah: On delete, clear host vitals.
+
+// NOTE: ADE lifecycle events are part of the integration_mdm_dep_test.go file
+
+type mdmLifecycleAssertion[T any] func(t *testing.T, host *fleet.Host, device T)
+
+func (s *integrationMDMTestSuite) TestTurnOnLifecycleEventsApple() {
+ t := s.T()
+ s.setupLifecycleSettings()
+
+ testCases := []struct {
+ Name string
+ Action mdmLifecycleAssertion[*mdmtest.TestAppleMDMClient]
+ }{
+ {
+ "wiped host turns on MDM",
+ func(t *testing.T, host *fleet.Host, device *mdmtest.TestAppleMDMClient) {
+ s.Do(
+ "POST",
+ fmt.Sprintf("/api/latest/fleet/hosts/%d/wipe", host.ID),
+ nil,
+ http.StatusNoContent,
+ )
+
+ cmd, err := device.Idle()
+ require.NoError(t, err)
+ for cmd != nil {
+ cmd, err = device.Acknowledge(cmd.CommandUUID)
+ require.NoError(t, err)
+ }
+
+ require.NoError(t, device.Enroll())
+ },
+ },
+ {
+ "locked host turns on MDM",
+ func(t *testing.T, host *fleet.Host, device *mdmtest.TestAppleMDMClient) {
+ s.Do(
+ "POST",
+ fmt.Sprintf("/api/latest/fleet/hosts/%d/lock", host.ID),
+ nil,
+ http.StatusNoContent,
+ )
+
+ cmd, err := device.Idle()
+ require.NoError(t, err)
+ for cmd != nil {
+ cmd, err = device.Acknowledge(cmd.CommandUUID)
+ require.NoError(t, err)
+ }
+
+ require.NoError(t, device.Enroll())
+ },
+ },
+ {
+ "host turns on MDM features out of the blue",
+ func(t *testing.T, host *fleet.Host, device *mdmtest.TestAppleMDMClient) {
+ require.NoError(t, device.Enroll())
+ },
+ },
+ {
+ "IT admin turns off MDM for a host via the UI then host turns on MDM",
+ func(t *testing.T, host *fleet.Host, device *mdmtest.TestAppleMDMClient) {
+ originalPushMock := s.pushProvider.PushFunc
+ defer func() { s.pushProvider.PushFunc = originalPushMock }()
+
+ s.pushProvider.PushFunc = func(pushes []*mdm.Push) (map[string]*push.Response, error) {
+ res, err := mockSuccessfulPush(pushes)
+ require.NoError(t, err)
+ err = device.Checkout()
+ require.NoError(t, err)
+ return res, err
+ }
+
+ s.Do(
+ "DELETE",
+ fmt.Sprintf("/api/latest/fleet/hosts/%d/mdm", host.ID),
+ nil,
+ http.StatusOK,
+ )
+
+ require.NoError(t, device.Enroll())
+ },
+ },
+ {
+ "host is deleted then turns on MDM",
+ func(t *testing.T, host *fleet.Host, device *mdmtest.TestAppleMDMClient) {
+ var delResp deleteHostResponse
+ s.DoJSON(
+ "DELETE",
+ fmt.Sprintf("/api/latest/fleet/hosts/%d", host.ID),
+ nil,
+ http.StatusOK,
+ &delResp,
+ )
+
+ dupeClient := mdmtest.NewTestMDMClientAppleDirect(
+ mdmtest.AppleEnrollInfo{
+
+ SCEPChallenge: s.fleetCfg.MDM.AppleSCEPChallenge,
+ SCEPURL: s.server.URL + apple_mdm.SCEPPath,
+ MDMURL: s.server.URL + apple_mdm.MDMPath,
+ },
+ )
+ dupeClient.UUID = device.UUID
+ dupeClient.SerialNumber = device.SerialNumber
+ dupeClient.Model = device.Model
+ require.NoError(t, dupeClient.Enroll())
+
+ *device = *dupeClient
+ },
+ },
+ {
+ "host is deleted in bulk then turns on MDM",
+ func(t *testing.T, host *fleet.Host, device *mdmtest.TestAppleMDMClient) {
+ req := deleteHostsRequest{
+ IDs: []uint{host.ID},
+ }
+ resp := deleteHostsResponse{}
+ s.DoJSON("POST", "/api/latest/fleet/hosts/delete", req, http.StatusOK, &resp)
+
+ dupeClient := mdmtest.NewTestMDMClientAppleDirect(
+ mdmtest.AppleEnrollInfo{
+
+ SCEPChallenge: s.fleetCfg.MDM.AppleSCEPChallenge,
+ SCEPURL: s.server.URL + apple_mdm.SCEPPath,
+ MDMURL: s.server.URL + apple_mdm.MDMPath,
+ },
+ )
+ dupeClient.UUID = device.UUID
+ dupeClient.SerialNumber = device.SerialNumber
+ dupeClient.Model = device.Model
+ require.NoError(t, dupeClient.Enroll())
+
+ *device = *dupeClient
+ },
+ },
+ {
+ "host is deleted then osquery enrolls then turns on MDM",
+ func(t *testing.T, host *fleet.Host, device *mdmtest.TestAppleMDMClient) {
+ var delResp deleteHostResponse
+ s.DoJSON(
+ "DELETE",
+ fmt.Sprintf("/api/latest/fleet/hosts/%d", host.ID),
+ nil,
+ http.StatusOK,
+ &delResp,
+ )
+
+ var err error
+ host.OsqueryHostID = ptr.String(t.Name())
+ host, err = s.ds.NewHost(context.Background(), host)
+ require.NoError(t, err)
+
+ setOrbitEnrollment(t, host, s.ds)
+ deviceToken := uuid.NewString()
+ err = s.ds.SetOrUpdateDeviceAuthToken(context.Background(), host.ID, deviceToken)
+ require.NoError(t, err)
+
+ device.SetDesktopToken(deviceToken)
+ require.NoError(t, device.Enroll())
+ },
+ },
+ }
+
+ assertAction := func(t *testing.T, host *fleet.Host, device *mdmtest.TestAppleMDMClient, action mdmLifecycleAssertion[*mdmtest.TestAppleMDMClient]) {
+ fCmds, fSumm, fHostMDM := s.recordAppleHostStatus(host, device)
+
+ action(t, host, device)
+
+ // reload the host by identifier, tests might
+ // delete hosts and create new records with different IDs
+ var err error
+ host, err = s.ds.HostByIdentifier(context.Background(), host.UUID)
+ require.NoError(t, err)
+
+ sCmds, sSumm, sHostMDM := s.recordAppleHostStatus(host, device)
+
+ // post asssertions
+ require.ElementsMatch(t, fCmds, sCmds)
+ require.Equal(t, fSumm, sSumm)
+ require.Equal(t, fHostMDM, sHostMDM)
+ }
+
+ for _, tt := range testCases {
+ t.Run(tt.Name, func(t *testing.T) {
+ t.Run("manual enrollment", func(t *testing.T) {
+ host, device := createHostThenEnrollMDM(s.ds, s.server.URL, t)
+ assertAction(t, host, device, tt.Action)
+ })
+
+ t.Run("automatic enrollment", func(t *testing.T) {
+ device := mdmtest.NewTestMDMClientAppleDEP(s.server.URL, "")
+ s.mockDEPResponse(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ encoder := json.NewEncoder(w)
+ switch r.URL.Path {
+ case "/session":
+ _, _ = w.Write([]byte(`{"auth_session_token": "xyz"}`))
+ case "/profile":
+ err := encoder.Encode(godep.ProfileResponse{ProfileUUID: "abc"})
+ require.NoError(t, err)
+ case "/profile/devices":
+ err := encoder.Encode(godep.ProfileResponse{
+ ProfileUUID: "abc",
+ Devices: map[string]string{},
+ })
+ require.NoError(t, err)
+ case "/server/devices", "/devices/sync":
+ err := encoder.Encode(godep.DeviceResponse{
+ Devices: []godep.Device{
+ {
+ SerialNumber: device.SerialNumber,
+ Model: device.Model,
+ OS: "osx",
+ OpType: "added",
+ },
+ },
+ })
+ require.NoError(t, err)
+ }
+ }))
+
+ s.runDEPSchedule()
+ depURLToken := loadEnrollmentProfileDEPToken(t, s.ds)
+ device.SetDEPToken(depURLToken)
+ var err error
+ host, err := s.ds.HostByIdentifier(context.Background(), device.SerialNumber)
+ require.NoError(t, err)
+ require.NoError(t, device.Enroll())
+
+ assertAction(t, host, device, tt.Action)
+ })
+ })
+ }
+}
+
+func (s *integrationMDMTestSuite) TestTurnOnLifecycleEventsWindows() {
+ t := s.T()
+ s.setupLifecycleSettings()
+
+ testCases := []struct {
+ Name string
+ Action mdmLifecycleAssertion[*mdmtest.TestWindowsMDMClient]
+ }{
+ {
+ "wiped host turns on MDM",
+ func(t *testing.T, host *fleet.Host, device *mdmtest.TestWindowsMDMClient) {
+ s.Do(
+ "POST",
+ fmt.Sprintf("/api/latest/fleet/hosts/%d/wipe", host.ID),
+ nil,
+ http.StatusNoContent,
+ )
+
+ status, err := s.ds.GetHostLockWipeStatus(context.Background(), host)
+ require.NoError(t, err)
+
+ cmds, err := device.StartManagementSession()
+ require.NoError(t, err)
+
+ // two status + the wipe command we enqueued
+ require.Len(t, cmds, 3)
+ wipeCmd := cmds[status.WipeMDMCommand.CommandUUID]
+ require.NotNil(t, wipeCmd)
+ require.Equal(t, wipeCmd.Verb, fleet.CmdExec)
+ require.Len(t, wipeCmd.Cmd.Items, 1)
+ require.EqualValues(t, "./Device/Vendor/MSFT/RemoteWipe/doWipeProtected", *wipeCmd.Cmd.Items[0].Target)
+
+ msgID, err := device.GetCurrentMsgID()
+ require.NoError(t, err)
+
+ device.AppendResponse(fleet.SyncMLCmd{
+ XMLName: xml.Name{Local: fleet.CmdStatus},
+ MsgRef: &msgID,
+ CmdRef: &status.WipeMDMCommand.CommandUUID,
+ Cmd: ptr.String("Exec"),
+ Data: ptr.String("200"),
+ Items: nil,
+ CmdID: fleet.CmdID{Value: uuid.NewString()},
+ })
+ cmds, err = device.SendResponse()
+ require.NoError(t, err)
+ // the ack of the message should be the only returned command
+ require.Len(t, cmds, 1)
+
+ // re-enroll
+ require.NoError(t, device.Enroll())
+ },
+ },
+ {
+ "locked host turns on MDM",
+ func(t *testing.T, host *fleet.Host, device *mdmtest.TestWindowsMDMClient) {
+ s.Do(
+ "POST",
+ fmt.Sprintf("/api/latest/fleet/hosts/%d/lock", host.ID),
+ nil,
+ http.StatusNoContent,
+ )
+
+ status, err := s.ds.GetHostLockWipeStatus(context.Background(), host)
+ require.NoError(t, err)
+
+ var orbitScriptResp orbitPostScriptResultResponse
+ s.DoJSON(
+ "POST",
+ "/api/fleet/orbit/scripts/result",
+ json.RawMessage(
+ fmt.Sprintf(
+ `{"orbit_node_key": %q, "execution_id": %q, "exit_code": 0, "output": "ok"}`,
+ *host.OrbitNodeKey,
+ status.LockScript.ExecutionID,
+ ),
+ ),
+ http.StatusOK,
+ &orbitScriptResp,
+ )
+
+ require.NoError(t, device.Enroll())
+ },
+ },
+ {
+ "host turns on MDM features out of the blue",
+ func(t *testing.T, host *fleet.Host, device *mdmtest.TestWindowsMDMClient) {
+ require.NoError(t, device.Enroll())
+ },
+ },
+ {
+ "host is deleted then osquery enrolls then turns on MDM",
+ func(t *testing.T, host *fleet.Host, device *mdmtest.TestWindowsMDMClient) {
+ var delResp deleteHostResponse
+ s.DoJSON(
+ "DELETE",
+ fmt.Sprintf("/api/latest/fleet/hosts/%d", host.ID),
+ nil,
+ http.StatusOK,
+ &delResp,
+ )
+
+ var err error
+ host.OsqueryHostID = ptr.String(t.Name())
+ host, err = s.ds.NewHost(context.Background(), host)
+ require.NoError(t, err)
+
+ orbitKey := setOrbitEnrollment(t, host, s.ds)
+ host.OrbitNodeKey = &orbitKey
+ if !strings.Contains(device.TokenIdentifier, "@") {
+ device.TokenIdentifier = orbitKey
+ }
+ device.HardwareID = host.UUID
+ device.DeviceID = host.UUID
+
+ require.NoError(t, device.Enroll())
+ },
+ },
+ }
+
+ assertAction := func(t *testing.T, host *fleet.Host, device *mdmtest.TestWindowsMDMClient, action mdmLifecycleAssertion[*mdmtest.TestWindowsMDMClient]) {
+ fCmds, fSumm, fHostMDM := s.recordWindowsHostStatus(host, device)
+
+ action(t, host, device)
+
+ // reload the host by identifier, tests might
+ // delete hosts and create new records with different IDs
+ var err error
+ host, err = s.ds.HostByIdentifier(context.Background(), host.UUID)
+ require.NoError(t, err)
+
+ sCmds, sSumm, sHostMDM := s.recordWindowsHostStatus(host, device)
+
+ // post asssertions
+ require.Len(t, sCmds, len(fCmds))
+ require.ElementsMatch(t, fCmds, sCmds)
+ require.Equal(t, fSumm, sSumm)
+ require.Equal(t, fHostMDM, sHostMDM)
+ }
+
+ for _, tt := range testCases {
+ t.Run(tt.Name, func(t *testing.T) {
+ t.Run("programmatic enrollment", func(t *testing.T) {
+ host, device := createWindowsHostThenEnrollMDM(s.ds, s.server.URL, t)
+ err := s.ds.SetOrUpdateMDMData(context.Background(), host.ID, false, true, s.server.URL, false, fleet.WellKnownMDMFleet, "")
+ require.NoError(t, err)
+ assertAction(t, host, device, tt.Action)
+ })
+
+ t.Run("automatic enrollment", func(t *testing.T) {
+ if strings.Contains(tt.Name, "wipe") {
+ t.Skip("wipe tests are not supported for windows automatic enrollment until we fix #TODO")
+ }
+
+ err := s.ds.ApplyEnrollSecrets(context.Background(), nil, []*fleet.EnrollSecret{{Secret: t.Name()}})
+ require.NoError(t, err)
+
+ host := createOrbitEnrolledHost(t, "windows", "windows_automatic", s.ds)
+
+ azureMail := "foo.bar.baz@example.com"
+ device := mdmtest.NewTestMDMClientWindowsAutomatic(s.server.URL, azureMail)
+ device.HardwareID = host.UUID
+ device.DeviceID = host.UUID
+ require.NoError(t, device.Enroll())
+
+ err = s.ds.SetOrUpdateMDMData(context.Background(), host.ID, false, true, s.server.URL, false, fleet.WellKnownMDMFleet, "")
+ require.NoError(t, err)
+
+ assertAction(t, host, device, tt.Action)
+ })
+ })
+ }
+}
+
+// Hardcode response type because we are using a custom json marshaling so
+// using getHostMDMResponse fails with "JSON unmarshaling is not supported for HostMDM".
+type jsonMDM struct {
+ EnrollmentStatus string `json:"enrollment_status"`
+ ServerURL string `json:"server_url"`
+ Name string `json:"name,omitempty"`
+ ID *uint `json:"id,omitempty"`
+}
+type getHostMDMResponseTest struct {
+ HostMDM *jsonMDM
+ Err error `json:"error,omitempty"`
+}
+
+func (s *integrationMDMTestSuite) recordWindowsHostStatus(
+ host *fleet.Host,
+ device *mdmtest.TestWindowsMDMClient,
+) ([]fleet.ProtoCmdOperation, getHostMDMSummaryResponse, getHostMDMResponseTest) {
+ t := s.T()
+
+ var recordedCmds []fleet.ProtoCmdOperation
+ cmds, err := device.StartManagementSession()
+ require.NoError(t, err)
+
+ msgID, err := device.GetCurrentMsgID()
+ require.NoError(t, err)
+ for _, c := range cmds {
+ cmdID := c.Cmd.CmdID
+ status := syncml.CmdStatusOK
+ device.AppendResponse(fleet.SyncMLCmd{
+ XMLName: xml.Name{Local: fleet.CmdStatus},
+ MsgRef: &msgID,
+ CmdRef: &cmdID.Value,
+ Cmd: ptr.String(c.Verb),
+ Data: &status,
+ Items: nil,
+ CmdID: fleet.CmdID{Value: uuid.NewString()},
+ })
+ c.Cmd.CmdID.Value = ""
+ c.Cmd.CmdRef = nil
+ recordedCmds = append(recordedCmds, c)
+ }
+
+ _, err = device.SendResponse()
+ require.NoError(t, err)
+
+ mdmAgg := getHostMDMSummaryResponse{}
+ s.DoJSON("GET", "/api/latest/fleet/hosts/summary/mdm", nil, http.StatusOK, &mdmAgg)
+
+ ghr := getHostMDMResponseTest{}
+ s.DoJSON(
+ "GET",
+ fmt.Sprintf("/api/latest/fleet/hosts/%d/mdm", host.ID),
+ nil,
+ http.StatusOK,
+ &ghr,
+ )
+
+ return recordedCmds, mdmAgg, ghr
+}
+
+func (s *integrationMDMTestSuite) recordAppleHostStatus(
+ host *fleet.Host,
+ device *mdmtest.TestAppleMDMClient,
+) ([]*micromdm.CommandPayload, getHostMDMSummaryResponse, getHostMDMResponseTest) {
+ t := s.T()
+
+ s.runWorker()
+ s.awaitTriggerProfileSchedule(t)
+
+ var cmds []*micromdm.CommandPayload
+
+ cmd, err := device.Idle()
+ require.NoError(t, err)
+ for cmd != nil {
+ var fullCmd micromdm.CommandPayload
+ require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd))
+
+ // command uuid is a random value, we only care that's set
+ require.NotEmpty(t, fullCmd.CommandUUID)
+ fullCmd.CommandUUID = ""
+
+ // strip the signature of the profiles so they can be easily compared
+ if fullCmd.Command.RequestType == "InstallProfile" {
+ p7, err := pkcs7.Parse(fullCmd.Command.InstallProfile.Payload)
+ require.NoError(t, err)
+ fullCmd.Command.InstallProfile.Payload = p7.Content
+ }
+ cmds = append(cmds, &fullCmd)
+
+ cmd, err = device.Acknowledge(cmd.CommandUUID)
+ require.NoError(t, err)
+ }
+
+ mdmAgg := getHostMDMSummaryResponse{}
+ s.DoJSON("GET", "/api/latest/fleet/hosts/summary/mdm", nil, http.StatusOK, &mdmAgg)
+
+ ghr := getHostMDMResponseTest{}
+ s.DoJSON(
+ "GET",
+ fmt.Sprintf("/api/latest/fleet/hosts/%d/mdm", host.ID),
+ nil,
+ http.StatusOK,
+ &ghr,
+ )
+
+ return cmds, mdmAgg, ghr
+}
+
+func (s *integrationMDMTestSuite) setupLifecycleSettings() {
+ t := s.T()
+ ctx := context.Background()
+ // add bootstrap package
+ _ = s.ds.DeleteMDMAppleBootstrapPackage(ctx, 0)
+ bp, err := os.ReadFile(filepath.Join("testdata", "bootstrap-packages", "signed.pkg"))
+ require.NoError(t, err)
+ s.uploadBootstrapPackage(
+ &fleet.MDMAppleBootstrapPackage{Bytes: bp, Name: "pkg.pkg", TeamID: 0},
+ http.StatusOK,
+ "",
+ )
+
+ // enable disk encryption
+ acResp := appConfigResponse{}
+ s.DoJSON("PATCH", "/api/latest/fleet/config", json.RawMessage(`{
+ "mdm": { "macos_settings": {"enable_disk_encryption": true} }
+ }`), http.StatusOK, &acResp)
+ require.True(t, acResp.MDM.EnableDiskEncryption.Value)
+
+ // add profiles (windows, mac)
+ s.Do(
+ "POST",
+ "/api/v1/fleet/mdm/profiles/batch",
+ batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{
+ {Name: "N1", Contents: mobileconfigForTest("N1", "I1")},
+ {Name: "N2", Contents: syncMLForTest("./Foo/Bar")},
+ {Name: "N3", Contents: declarationForTest("D1")},
+ }},
+ http.StatusNoContent,
+ )
+}
+
+// Host is renewing SCEP certificates
+func (s *integrationMDMTestSuite) TestLifecycleSCEPCertExpiration() {
+ t := s.T()
+ ctx := context.Background()
+ // ensure there's a token for automatic enrollments
+ s.mockDEPResponse(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(`{"auth_session_token": "xyz"}`))
+ }))
+ s.runDEPSchedule()
+
+ // add a device that's manually enrolled
+ desktopToken := uuid.New().String()
+ manualHost := createOrbitEnrolledHost(t, "darwin", "h1", s.ds)
+ err := s.ds.SetOrUpdateDeviceAuthToken(context.Background(), manualHost.ID, desktopToken)
+ require.NoError(t, err)
+ manualEnrolledDevice := mdmtest.NewTestMDMClientAppleDesktopManual(s.server.URL, desktopToken)
+ manualEnrolledDevice.UUID = manualHost.UUID
+ err = manualEnrolledDevice.Enroll()
+ require.NoError(t, err)
+
+ // add a device that's automatically enrolled
+ automaticHost := createOrbitEnrolledHost(t, "darwin", "h2", s.ds)
+ depURLToken := loadEnrollmentProfileDEPToken(t, s.ds)
+ automaticEnrolledDevice := mdmtest.NewTestMDMClientAppleDEP(s.server.URL, depURLToken)
+ automaticEnrolledDevice.UUID = automaticHost.UUID
+ automaticEnrolledDevice.SerialNumber = automaticHost.HardwareSerial
+ err = automaticEnrolledDevice.Enroll()
+ require.NoError(t, err)
+
+ // add a device that's automatically enrolled with a server ref
+ automaticHostWithRef := createOrbitEnrolledHost(t, "darwin", "h3", s.ds)
+ automaticEnrolledDeviceWithRef := mdmtest.NewTestMDMClientAppleDEP(s.server.URL, depURLToken)
+ automaticEnrolledDeviceWithRef.UUID = automaticHostWithRef.UUID
+ automaticEnrolledDeviceWithRef.SerialNumber = automaticHostWithRef.HardwareSerial
+ err = automaticEnrolledDeviceWithRef.Enroll()
+ require.NoError(
+ t,
+ s.ds.SetOrUpdateMDMData(
+ ctx,
+ automaticHostWithRef.ID,
+ false,
+ true,
+ s.server.URL,
+ true,
+ fleet.WellKnownMDMFleet,
+ "foo",
+ ),
+ )
+ require.NoError(t, err)
+
+ // add global profiles
+ globalProfiles := [][]byte{
+ mobileconfigForTest("N1", "I1"),
+ mobileconfigForTest("N2", "I2"),
+ }
+ s.Do(
+ "POST",
+ "/api/v1/fleet/mdm/apple/profiles/batch",
+ batchSetMDMAppleProfilesRequest{Profiles: globalProfiles},
+ http.StatusNoContent,
+ )
+ // ack all commands to install profiles
+ cmd, err := manualEnrolledDevice.Idle()
+ require.NoError(t, err)
+ for cmd != nil {
+ cmd, err = manualEnrolledDevice.Acknowledge(cmd.CommandUUID)
+ require.NoError(t, err)
+ }
+ cmd, err = automaticEnrolledDevice.Idle()
+ require.NoError(t, err)
+ for cmd != nil {
+ cmd, err = automaticEnrolledDevice.Acknowledge(cmd.CommandUUID)
+ require.NoError(t, err)
+ }
+ cmd, err = automaticEnrolledDeviceWithRef.Idle()
+ require.NoError(t, err)
+ for cmd != nil {
+ cmd, err = automaticEnrolledDeviceWithRef.Acknowledge(cmd.CommandUUID)
+ require.NoError(t, err)
+ }
+
+ cert, key, err := generateCertWithAPNsTopic()
+ require.NoError(t, err)
+ fleetCfg := config.TestConfig()
+ config.SetTestMDMConfig(s.T(), &fleetCfg, cert, key, testBMToken, "")
+ logger := kitlog.NewJSONLogger(os.Stdout)
+
+ // run without expired certs, no command enqueued
+ err = RenewSCEPCertificates(ctx, logger, s.ds, &fleetCfg, s.mdmCommander)
+ require.NoError(t, err)
+ cmd, err = manualEnrolledDevice.Idle()
+ require.NoError(t, err)
+ require.Nil(t, cmd)
+
+ cmd, err = automaticEnrolledDevice.Idle()
+ require.NoError(t, err)
+ require.Nil(t, cmd)
+
+ cmd, err = automaticEnrolledDeviceWithRef.Idle()
+ require.NoError(t, err)
+ require.Nil(t, cmd)
+
+ // expire all the certs we just created
+ mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error {
+ _, err := q.ExecContext(ctx, `
+ UPDATE nano_cert_auth_associations
+ SET cert_not_valid_after = DATE_SUB(CURDATE(), INTERVAL 1 YEAR)
+ WHERE id IN (?, ?, ?)
+ `, manualHost.UUID, automaticHost.UUID, automaticHostWithRef.UUID)
+ return err
+ })
+
+ // generate a new config here so we can manipulate the certs.
+ err = RenewSCEPCertificates(ctx, logger, s.ds, &fleetCfg, s.mdmCommander)
+ require.NoError(t, err)
+
+ checkRenewCertCommand := func(device *mdmtest.TestAppleMDMClient, enrollRef string) {
+ var renewCmd *mdm.Command
+ cmd, err := device.Idle()
+ require.NoError(t, err)
+ for cmd != nil {
+ if cmd.Command.RequestType == "InstallProfile" {
+ renewCmd = cmd
+ }
+ cmd, err = device.Acknowledge(cmd.CommandUUID)
+ require.NoError(t, err)
+ }
+ require.NotNil(t, renewCmd)
+ var fullCmd micromdm.CommandPayload
+ require.NoError(t, plist.Unmarshal(renewCmd.Raw, &fullCmd))
+ s.verifyEnrollmentProfile(fullCmd.Command.InstallProfile.Payload, enrollRef)
+ }
+
+ checkRenewCertCommand(manualEnrolledDevice, "")
+ checkRenewCertCommand(automaticEnrolledDevice, "")
+ checkRenewCertCommand(automaticEnrolledDeviceWithRef, "foo")
+
+ // another cron run shouldn't enqueue more commands
+ err = RenewSCEPCertificates(ctx, logger, s.ds, &fleetCfg, s.mdmCommander)
+ require.NoError(t, err)
+
+ cmd, err = manualEnrolledDevice.Idle()
+ require.NoError(t, err)
+ require.Nil(t, cmd)
+
+ cmd, err = automaticEnrolledDevice.Idle()
+ require.NoError(t, err)
+ require.Nil(t, cmd)
+
+ cmd, err = automaticEnrolledDeviceWithRef.Idle()
+ require.NoError(t, err)
+ require.Nil(t, cmd)
+
+ // devices renew their SCEP cert by re-enrolling.
+ require.NoError(t, manualEnrolledDevice.Enroll())
+ require.NoError(t, automaticEnrolledDevice.Enroll())
+ require.NoError(t, automaticEnrolledDeviceWithRef.Enroll())
+
+ // no new commands are enqueued right after enrollment
+ cmd, err = manualEnrolledDevice.Idle()
+ require.NoError(t, err)
+ require.Nil(t, cmd)
+
+ cmd, err = automaticEnrolledDevice.Idle()
+ require.NoError(t, err)
+ require.Nil(t, cmd)
+
+ cmd, err = automaticEnrolledDeviceWithRef.Idle()
+ require.NoError(t, err)
+ require.Nil(t, cmd)
+}
diff --git a/server/service/integration_mdm_profiles_test.go b/server/service/integration_mdm_profiles_test.go
index 0b10dbeb4c..303d147abf 100644
--- a/server/service/integration_mdm_profiles_test.go
+++ b/server/service/integration_mdm_profiles_test.go
@@ -1983,14 +1983,20 @@ func (s *integrationMDMTestSuite) TestHostMDMAppleProfilesStatus() {
require.Nil(t, h1.TeamID)
h2 := createManualMDMEnrollWithOrbit(globalEnrollSec)
require.Nil(t, h2.TeamID)
+ // run the cron
+ s.awaitTriggerProfileSchedule(t)
s.assertHostConfigProfiles(map[*fleet.Host][]fleet.HostMDMAppleProfile{
h1: {
{Identifier: "G1", OperationType: fleet.MDMOperationTypeInstall, Status: &fleet.MDMDeliveryPending},
{Identifier: "G2", OperationType: fleet.MDMOperationTypeInstall, Status: &fleet.MDMDeliveryPending},
+ {Identifier: mobileconfig.FleetdConfigPayloadIdentifier, OperationType: fleet.MDMOperationTypeInstall, Status: &fleet.MDMDeliveryPending},
+ {Identifier: mobileconfig.FleetCARootConfigPayloadIdentifier, OperationType: fleet.MDMOperationTypeInstall, Status: &fleet.MDMDeliveryPending},
},
h2: {
{Identifier: "G1", OperationType: fleet.MDMOperationTypeInstall, Status: &fleet.MDMDeliveryPending},
{Identifier: "G2", OperationType: fleet.MDMOperationTypeInstall, Status: &fleet.MDMDeliveryPending},
+ {Identifier: mobileconfig.FleetdConfigPayloadIdentifier, OperationType: fleet.MDMOperationTypeInstall, Status: &fleet.MDMDeliveryPending},
+ {Identifier: mobileconfig.FleetCARootConfigPayloadIdentifier, OperationType: fleet.MDMOperationTypeInstall, Status: &fleet.MDMDeliveryPending},
},
})
@@ -2001,14 +2007,20 @@ func (s *integrationMDMTestSuite) TestHostMDMAppleProfilesStatus() {
h4 := createManualMDMEnrollWithOrbit(tm1EnrollSec)
require.NotNil(t, h4.TeamID)
require.Equal(t, tm1.ID, *h4.TeamID)
+ // run the cron
+ s.awaitTriggerProfileSchedule(t)
s.assertHostConfigProfiles(map[*fleet.Host][]fleet.HostMDMAppleProfile{
h3: {
{Identifier: "T1.1", OperationType: fleet.MDMOperationTypeInstall, Status: &fleet.MDMDeliveryPending},
{Identifier: "T1.2", OperationType: fleet.MDMOperationTypeInstall, Status: &fleet.MDMDeliveryPending},
+ {Identifier: mobileconfig.FleetdConfigPayloadIdentifier, OperationType: fleet.MDMOperationTypeInstall, Status: &fleet.MDMDeliveryPending},
+ {Identifier: mobileconfig.FleetCARootConfigPayloadIdentifier, OperationType: fleet.MDMOperationTypeInstall, Status: &fleet.MDMDeliveryPending},
},
h4: {
{Identifier: "T1.1", OperationType: fleet.MDMOperationTypeInstall, Status: &fleet.MDMDeliveryPending},
{Identifier: "T1.2", OperationType: fleet.MDMOperationTypeInstall, Status: &fleet.MDMDeliveryPending},
+ {Identifier: mobileconfig.FleetdConfigPayloadIdentifier, OperationType: fleet.MDMOperationTypeInstall, Status: &fleet.MDMDeliveryPending},
+ {Identifier: mobileconfig.FleetCARootConfigPayloadIdentifier, OperationType: fleet.MDMOperationTypeInstall, Status: &fleet.MDMDeliveryPending},
},
})
@@ -4001,9 +4013,7 @@ func (s *integrationMDMTestSuite) TestMDMBatchSetProfilesKeepsReservedNames() {
if len(secrets) == 0 {
require.NoError(t, s.ds.ApplyEnrollSecrets(ctx, nil, []*fleet.EnrollSecret{{Secret: t.Name()}}))
}
- signingCert, _, _, err := s.fleetCfg.MDM.AppleSCEP()
- require.NoError(t, err)
- require.NoError(t, ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, s.logger, signingCert))
+ require.NoError(t, ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, s.logger, s.fleetCfg.MDM))
// turn on disk encryption and os updates
s.DoJSON("PATCH", "/api/latest/fleet/config", json.RawMessage(`{
@@ -4081,7 +4091,7 @@ func (s *integrationMDMTestSuite) TestMDMBatchSetProfilesKeepsReservedNames() {
require.Equal(t, "2023-12-31", tmResp.Team.Config.MDM.MacOSUpdates.Deadline.Value)
require.Equal(t, "13.3.8", tmResp.Team.Config.MDM.MacOSUpdates.MinimumVersion.Value)
- require.NoError(t, ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, s.logger, signingCert))
+ require.NoError(t, ReconcileAppleProfiles(ctx, s.ds, s.mdmCommander, s.logger, s.fleetCfg.MDM))
checkMacProfs(&tmResp.Team.ID, servermdm.ListFleetReservedMacOSProfileNames()...)
checkWinProfs(&tmResp.Team.ID, servermdm.ListFleetReservedWindowsProfileNames()...)
diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go
index 90cb5fad04..7dc1ee4fce 100644
--- a/server/service/integration_mdm_test.go
+++ b/server/service/integration_mdm_test.go
@@ -212,9 +212,7 @@ func (s *integrationMDMTestSuite) SetupSuite() {
if s.onProfileJobDone != nil {
s.onProfileJobDone()
}
- signingCert, _, _, err := fleetCfg.MDM.AppleSCEP()
- require.NoError(s.T(), err)
- err = ReconcileAppleProfiles(ctx, ds, mdmCommander, logger, signingCert)
+ err = ReconcileAppleProfiles(ctx, ds, mdmCommander, logger, fleetCfg.MDM)
require.NoError(s.T(), err)
return err
}),
@@ -315,6 +313,8 @@ func (s *integrationMDMTestSuite) TearDownTest() {
appCfg.MDM.MacOSSetup.EnableReleaseDeviceManually = optjson.SetBool(false)
// ensure global Windows OS updates are always disabled for the next test
appCfg.MDM.WindowsUpdates = fleet.WindowsUpdates{}
+ // ensure the server URL is constant
+ appCfg.ServerSettings.ServerURL = s.server.URL
err := s.ds.SaveAppConfig(ctx, &appCfg.AppConfig)
require.NoError(t, err)
@@ -6071,22 +6071,79 @@ func (s *integrationMDMTestSuite) TestWindowsAutomaticEnrollmentCommands() {
d := mdmtest.NewTestMDMClientWindowsAutomatic(s.server.URL, azureMail)
require.NoError(t, d.Enroll())
- cmds, err := d.StartManagementSession()
+ checkinAndAck := func(expectFleetdCmds bool) {
+ cmds, err := d.StartManagementSession()
+ require.NoError(t, err)
+
+ if !expectFleetdCmds {
+ // receives only the 2 status commands
+ require.Len(t, cmds, 2)
+ for _, c := range cmds {
+ require.Equal(t, "Status", c.Verb, c)
+ }
+ return
+ }
+
+ // 2 status + 2 commands to install fleetd
+ require.Len(t, cmds, 4)
+ var fleetdAddCmd, fleetdExecCmd fleet.ProtoCmdOperation
+ for _, c := range cmds {
+ switch c.Verb {
+ case "Add":
+ fleetdAddCmd = c
+ case "Exec":
+ fleetdExecCmd = c
+ }
+ }
+ require.Equal(t, syncml.FleetdWindowsInstallerGUID, fleetdAddCmd.Cmd.GetTargetURI())
+ require.Equal(t, syncml.FleetdWindowsInstallerGUID, fleetdExecCmd.Cmd.GetTargetURI())
+
+ // reply with success for both commands
+ msgID, err := d.GetCurrentMsgID()
+ require.NoError(t, err)
+
+ d.AppendResponse(fleet.SyncMLCmd{
+ XMLName: xml.Name{Local: fleet.CmdStatus},
+ MsgRef: &msgID,
+ CmdRef: &fleetdAddCmd.Cmd.CmdID.Value,
+ Cmd: &fleetdAddCmd.Verb,
+ Data: ptr.String("200"),
+ Items: nil,
+ CmdID: fleet.CmdID{Value: uuid.NewString()},
+ })
+ d.AppendResponse(fleet.SyncMLCmd{
+ XMLName: xml.Name{Local: fleet.CmdStatus},
+ MsgRef: &msgID,
+ CmdRef: &fleetdExecCmd.Cmd.CmdID.Value,
+ Cmd: &fleetdExecCmd.Verb,
+ Data: ptr.String("200"),
+ Items: nil,
+ CmdID: fleet.CmdID{Value: uuid.NewString()},
+ })
+ cmds, err = d.SendResponse()
+ require.NoError(t, err)
+
+ // the ack of the message should be the only returned command
+ require.Len(t, cmds, 1)
+ }
+
+ // start a management session, will receive the install fleetd commands
+ checkinAndAck(true)
+
+ // start a new management session again, Fleetd is not reported as installed
+ // so it receives the commands again
+ checkinAndAck(true)
+
+ // simulate fleetd installed and enrolled
+ host := createOrbitEnrolledHost(t, "windows", "h1", s.ds)
+ err = s.ds.UpdateMDMWindowsEnrollmentsHostUUID(ctx, host.UUID, d.DeviceID)
+ require.NoError(t, err)
+ err = s.ds.SetOrUpdateHostOrbitInfo(ctx, host.ID, "1.23", sql.NullString{}, sql.NullBool{})
require.NoError(t, err)
- // 2 status + 2 commands to install fleetd
- require.Len(t, cmds, 4)
- var fleetdAddCmd, fleetdExecCmd fleet.ProtoCmdOperation
- for _, c := range cmds {
- switch c.Verb {
- case "Add":
- fleetdAddCmd = c
- case "Exec":
- fleetdExecCmd = c
- }
- }
- require.Equal(t, syncml.FleetdWindowsInstallerGUID, fleetdAddCmd.Cmd.GetTargetURI())
- require.Equal(t, syncml.FleetdWindowsInstallerGUID, fleetdExecCmd.Cmd.GetTargetURI())
+ // start a new management session again, Fleetd is reported as installed so
+ // it does not receive the commands
+ checkinAndAck(false)
}
func (s *integrationMDMTestSuite) TestValidManagementUnenrollRequest() {
@@ -7525,7 +7582,6 @@ func (s *integrationMDMTestSuite) checkMDMProfilesSummaries(t *testing.T, teamID
if expectedAppleSummary != nil {
var apple getMDMAppleProfilesSummaryResponse
s.DoJSON("GET", "/api/v1/fleet/mdm/apple/profiles/summary", getMDMAppleProfilesSummaryRequest{}, http.StatusOK, &apple, queryParams...)
- fmt.Println(expectedSummary, apple)
require.Equal(t, expectedSummary.Failed, apple.Failed, "failed summary count doesn't match")
require.Equal(t, expectedSummary.Pending, apple.Pending, "pending summary count doesn't match")
require.Equal(t, expectedSummary.Verifying, apple.Verifying, "verifying summary count doesn't match")
@@ -7637,7 +7693,7 @@ func (s *integrationMDMTestSuite) TestManualEnrollmentCommands() {
checkInstallFleetdCommandSent(mdmDevice, true)
// create a device that's enrolled into Fleet before turning on MDM features,
- // it shouldn't get the command to install fleetd
+ // it should still get the command to install fleetd if turns on MDM.
desktopToken := uuid.New().String()
host := createOrbitEnrolledHost(t, "darwin", "h1", s.ds)
err = s.ds.SetOrUpdateDeviceAuthToken(context.Background(), host.ID, desktopToken)
@@ -7647,7 +7703,7 @@ func (s *integrationMDMTestSuite) TestManualEnrollmentCommands() {
err = mdmDevice.Enroll()
require.NoError(t, err)
s.runWorker()
- checkInstallFleetdCommandSent(mdmDevice, false)
+ checkInstallFleetdCommandSent(mdmDevice, true)
}
func (s *integrationMDMTestSuite) TestLockUnlockWipeWindowsLinux() {
@@ -8320,161 +8376,6 @@ func (s *integrationMDMTestSuite) TestDontIgnoreAnyProfileErrors() {
}
}
-func (s *integrationMDMTestSuite) TestSCEPCertExpiration() {
- t := s.T()
- ctx := context.Background()
- // ensure there's a token for automatic enrollments
- s.mockDEPResponse(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.WriteHeader(http.StatusOK)
- _, _ = w.Write([]byte(`{"auth_session_token": "xyz"}`))
- }))
- s.runDEPSchedule()
-
- // add a device that's manually enrolled
- desktopToken := uuid.New().String()
- manualHost := createOrbitEnrolledHost(t, "darwin", "h1", s.ds)
- err := s.ds.SetOrUpdateDeviceAuthToken(context.Background(), manualHost.ID, desktopToken)
- require.NoError(t, err)
- manualEnrolledDevice := mdmtest.NewTestMDMClientAppleDesktopManual(s.server.URL, desktopToken)
- manualEnrolledDevice.UUID = manualHost.UUID
- err = manualEnrolledDevice.Enroll()
- require.NoError(t, err)
-
- // add a device that's automatically enrolled
- automaticHost := createOrbitEnrolledHost(t, "darwin", "h2", s.ds)
- depURLToken := loadEnrollmentProfileDEPToken(t, s.ds)
- automaticEnrolledDevice := mdmtest.NewTestMDMClientAppleDEP(s.server.URL, depURLToken)
- automaticEnrolledDevice.UUID = automaticHost.UUID
- automaticEnrolledDevice.SerialNumber = automaticHost.HardwareSerial
- err = automaticEnrolledDevice.Enroll()
- require.NoError(t, err)
-
- // add a device that's automatically enrolled with a server ref
- automaticHostWithRef := createOrbitEnrolledHost(t, "darwin", "h3", s.ds)
- automaticEnrolledDeviceWithRef := mdmtest.NewTestMDMClientAppleDEP(s.server.URL, depURLToken)
- automaticEnrolledDeviceWithRef.UUID = automaticHostWithRef.UUID
- automaticEnrolledDeviceWithRef.SerialNumber = automaticHostWithRef.HardwareSerial
- err = automaticEnrolledDeviceWithRef.Enroll()
- require.NoError(t, s.ds.SetOrUpdateMDMData(ctx, automaticHostWithRef.ID, false, true, s.server.URL, true, fleet.WellKnownMDMFleet, "foo"))
- require.NoError(t, err)
-
- // add global profiles
- globalProfiles := [][]byte{
- mobileconfigForTest("N1", "I1"),
- mobileconfigForTest("N2", "I2"),
- }
- s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: globalProfiles}, http.StatusNoContent)
- // ack all commands to install profiles
- cmd, err := manualEnrolledDevice.Idle()
- require.NoError(t, err)
- for cmd != nil {
- cmd, err = manualEnrolledDevice.Acknowledge(cmd.CommandUUID)
- require.NoError(t, err)
- }
- cmd, err = automaticEnrolledDevice.Idle()
- require.NoError(t, err)
- for cmd != nil {
- cmd, err = automaticEnrolledDevice.Acknowledge(cmd.CommandUUID)
- require.NoError(t, err)
- }
- cmd, err = automaticEnrolledDeviceWithRef.Idle()
- require.NoError(t, err)
- for cmd != nil {
- cmd, err = automaticEnrolledDeviceWithRef.Acknowledge(cmd.CommandUUID)
- require.NoError(t, err)
- }
-
- cert, key, err := generateCertWithAPNsTopic()
- require.NoError(t, err)
- fleetCfg := config.TestConfig()
- config.SetTestMDMConfig(s.T(), &fleetCfg, cert, key, testBMToken, "")
- logger := kitlog.NewJSONLogger(os.Stdout)
-
- // run without expired certs, no command enqueued
- err = RenewSCEPCertificates(ctx, logger, s.ds, &fleetCfg, s.mdmCommander)
- require.NoError(t, err)
- cmd, err = manualEnrolledDevice.Idle()
- require.NoError(t, err)
- require.Nil(t, cmd)
-
- cmd, err = automaticEnrolledDevice.Idle()
- require.NoError(t, err)
- require.Nil(t, cmd)
-
- cmd, err = automaticEnrolledDeviceWithRef.Idle()
- require.NoError(t, err)
- require.Nil(t, cmd)
-
- // expire all the certs we just created
- mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error {
- _, err := q.ExecContext(ctx, `
- UPDATE nano_cert_auth_associations
- SET cert_not_valid_after = DATE_SUB(CURDATE(), INTERVAL 1 YEAR)
- WHERE id IN (?, ?, ?)
- `, manualHost.UUID, automaticHost.UUID, automaticHostWithRef.UUID)
- return err
- })
-
- // generate a new config here so we can manipulate the certs.
- err = RenewSCEPCertificates(ctx, logger, s.ds, &fleetCfg, s.mdmCommander)
- require.NoError(t, err)
-
- checkRenewCertCommand := func(device *mdmtest.TestAppleMDMClient, enrollRef string) {
- var renewCmd *mdm.Command
- cmd, err := device.Idle()
- require.NoError(t, err)
- for cmd != nil {
- if cmd.Command.RequestType == "InstallProfile" {
- renewCmd = cmd
- }
- cmd, err = device.Acknowledge(cmd.CommandUUID)
- require.NoError(t, err)
- }
- require.NotNil(t, renewCmd)
- var fullCmd micromdm.CommandPayload
- require.NoError(t, plist.Unmarshal(renewCmd.Raw, &fullCmd))
- s.verifyEnrollmentProfile(fullCmd.Command.InstallProfile.Payload, enrollRef)
- }
-
- checkRenewCertCommand(manualEnrolledDevice, "")
- checkRenewCertCommand(automaticEnrolledDevice, "")
- checkRenewCertCommand(automaticEnrolledDeviceWithRef, "foo")
-
- // another cron run shouldn't enqueue more commands
- err = RenewSCEPCertificates(ctx, logger, s.ds, &fleetCfg, s.mdmCommander)
- require.NoError(t, err)
-
- cmd, err = manualEnrolledDevice.Idle()
- require.NoError(t, err)
- require.Nil(t, cmd)
-
- cmd, err = automaticEnrolledDevice.Idle()
- require.NoError(t, err)
- require.Nil(t, cmd)
-
- cmd, err = automaticEnrolledDeviceWithRef.Idle()
- require.NoError(t, err)
- require.Nil(t, cmd)
-
- // devices renew their SCEP cert by re-enrolling.
- require.NoError(t, manualEnrolledDevice.Enroll())
- require.NoError(t, automaticEnrolledDevice.Enroll())
- require.NoError(t, automaticEnrolledDeviceWithRef.Enroll())
-
- // no new commands are enqueued right after enrollment
- cmd, err = manualEnrolledDevice.Idle()
- require.NoError(t, err)
- require.Nil(t, cmd)
-
- cmd, err = automaticEnrolledDevice.Idle()
- require.NoError(t, err)
- require.Nil(t, cmd)
-
- cmd, err = automaticEnrolledDeviceWithRef.Idle()
- require.NoError(t, err)
- require.Nil(t, cmd)
-}
-
func (s *integrationMDMTestSuite) TestMDMDiskEncryptionIssue16636() {
// see https://github.com/fleetdm/fleet/issues/16636
diff --git a/server/service/microsoft_mdm.go b/server/service/microsoft_mdm.go
index 1a6e113f50..e8a44ae0ae 100644
--- a/server/service/microsoft_mdm.go
+++ b/server/service/microsoft_mdm.go
@@ -23,6 +23,7 @@ import (
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
"github.com/fleetdm/fleet/v4/server/contexts/logging"
"github.com/fleetdm/fleet/v4/server/fleet"
+ mdmlifecycle "github.com/fleetdm/fleet/v4/server/mdm/lifecycle"
microsoft_mdm "github.com/fleetdm/fleet/v4/server/mdm/microsoft"
"github.com/fleetdm/fleet/v4/server/mdm/microsoft/syncml"
kitlog "github.com/go-kit/kit/log"
@@ -1274,7 +1275,23 @@ func (svc *Service) isFleetdPresentOnDevice(ctx context.Context, deviceID string
// If user identity is a MS-MDM UPN it means that the device was enrolled through user-driven flow
// This means that fleetd might not be installed
if isValidUPN(enrolledDevice.MDMEnrollUserID) {
- return false, nil
+ var isPresent bool
+ if enrolledDevice.HostUUID != "" {
+ host, err := svc.ds.HostLiteByIdentifier(ctx, enrolledDevice.HostUUID)
+ if err != nil && !fleet.IsNotFound(err) {
+ return false, ctxerr.Wrap(ctx, err, "get host lite by identifier")
+ }
+ if host != nil {
+ orbitInfo, err := svc.ds.GetHostOrbitInfo(ctx, host.ID)
+ if err != nil && !fleet.IsNotFound(err) {
+ return false, ctxerr.Wrap(ctx, err, "get host orbit info")
+ }
+ if orbitInfo != nil {
+ isPresent = orbitInfo.Version != ""
+ }
+ }
+ }
+ return isPresent, nil
}
// TODO: Add check here to determine if MDM DeviceID is connected with Smbios UUID present on
@@ -1746,6 +1763,20 @@ func (svc *Service) storeWindowsMDMEnrolledDevice(ctx context.Context, userID st
return err
}
+ // TODO: azure enrollments come with an empty uuid, I haven't figured
+ // out a good way to identify the device.
+ if hostUUID != "" {
+ mdmLifecycle := mdmlifecycle.New(svc.ds, svc.logger)
+ err = mdmLifecycle.Do(ctx, mdmlifecycle.HostOptions{
+ Action: mdmlifecycle.HostActionTurnOn,
+ Platform: "windows",
+ UUID: hostUUID,
+ })
+ if err != nil {
+ return err
+ }
+ }
+
err = svc.ds.NewActivity(ctx, nil, &fleet.ActivityTypeMDMEnrolled{
HostDisplayName: reqDeviceName,
MDMPlatform: fleet.MDMPlatformMicrosoft,
diff --git a/server/service/osquery_test.go b/server/service/osquery_test.go
index d6e34624f6..10387de164 100644
--- a/server/service/osquery_test.go
+++ b/server/service/osquery_test.go
@@ -259,7 +259,10 @@ var allDetailQueries = osquery_utils.GetDetailQueries(
context.Background(),
config.FleetConfig{Vulnerabilities: config.VulnerabilitiesConfig{DisableWinOSVulnerabilities: true}},
nil,
- &fleet.Features{EnableHostUsers: true},
+ &fleet.Features{
+ EnableHostUsers: true,
+ EnableSoftwareInventory: true,
+ },
)
func expectedDetailQueriesForPlatform(platform string) map[string]osquery_utils.DetailQuery {
@@ -1027,7 +1030,11 @@ func TestHostDetailQueries(t *testing.T) {
ds := new(mock.Store)
additional := json.RawMessage(`{"foobar": "select foo", "bim": "bam"}`)
ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
- return &fleet.AppConfig{Features: fleet.Features{AdditionalQueries: &additional, EnableHostUsers: true}}, nil
+ return &fleet.AppConfig{Features: fleet.Features{
+ AdditionalQueries: &additional,
+ EnableHostUsers: true,
+ EnableSoftwareInventory: true,
+ }}, nil
}
mockClock := clock.NewMockClock()
@@ -1311,7 +1318,10 @@ func TestLabelQueries(t *testing.T) {
return nil
}
ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
- return &fleet.AppConfig{Features: fleet.Features{EnableHostUsers: true}}, nil
+ return &fleet.AppConfig{Features: fleet.Features{
+ EnableHostUsers: true,
+ EnableSoftwareInventory: true,
+ }}, nil
}
ds.PolicyQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) {
return map[string]string{}, nil
@@ -1468,7 +1478,10 @@ func TestDetailQueriesWithEmptyStrings(t *testing.T) {
ctx = hostctx.NewContext(ctx, host)
ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
- return &fleet.AppConfig{Features: fleet.Features{EnableHostUsers: true}}, nil
+ return &fleet.AppConfig{Features: fleet.Features{
+ EnableHostUsers: true,
+ EnableSoftwareInventory: true,
+ }}, nil
}
ds.LabelQueriesForHostFunc = func(context.Context, *fleet.Host) (map[string]string, error) {
return map[string]string{}, nil
@@ -1658,7 +1671,10 @@ func TestDetailQueries(t *testing.T) {
lq.On("QueriesForHost", host.ID).Return(map[string]string{}, nil)
ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
- return &fleet.AppConfig{Features: fleet.Features{EnableHostUsers: true, EnableSoftwareInventory: true}}, nil
+ return &fleet.AppConfig{Features: fleet.Features{
+ EnableHostUsers: true,
+ EnableSoftwareInventory: true,
+ }}, nil
}
ds.LabelQueriesForHostFunc = func(context.Context, *fleet.Host) (map[string]string, error) {
return map[string]string{}, nil
@@ -1707,9 +1723,8 @@ func TestDetailQueries(t *testing.T) {
// queries)
queries, discovery, acc, err := svc.GetDistributedQueries(ctx)
require.NoError(t, err)
- // +2 for software inventory (+1 for the main software query +1 software_vscode_extensions)
// +1 for fleet_no_policies_wildcard
- if expected := expectedDetailQueriesForPlatform(host.Platform); !assert.Equal(t, len(expected)+2+1, len(queries)) {
+ if expected := expectedDetailQueriesForPlatform(host.Platform); !assert.Equal(t, len(expected)+1, len(queries)) {
// this is just to print the diff between the expected and actual query
// keys when the count assertion fails, to help debugging - they are not
// expected to match.
@@ -1975,9 +1990,8 @@ func TestDetailQueries(t *testing.T) {
queries, discovery, acc, err = svc.GetDistributedQueries(ctx)
require.NoError(t, err)
- // +2 software inventory (+1 main software query and +1 software extra query )
// +1 fleet_no_policies_wildcard query
- require.Equal(t, len(expectedDetailQueriesForPlatform(host.Platform))+2+1, len(queries), distQueriesMapKeys(queries))
+ require.Equal(t, len(expectedDetailQueriesForPlatform(host.Platform))+1, len(queries), distQueriesMapKeys(queries))
verifyDiscovery(t, queries, discovery)
assert.Zero(t, acc)
}
@@ -2156,7 +2170,10 @@ func TestDistributedQueryResults(t *testing.T) {
return nil
}
ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
- return &fleet.AppConfig{Features: fleet.Features{EnableHostUsers: true}}, nil
+ return &fleet.AppConfig{Features: fleet.Features{
+ EnableHostUsers: true,
+ EnableSoftwareInventory: true,
+ }}, nil
}
hostCtx := hostctx.NewContext(ctx, host)
@@ -3012,7 +3029,10 @@ func TestPolicyQueries(t *testing.T) {
return nil
}
ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
- return &fleet.AppConfig{Features: fleet.Features{EnableHostUsers: true}}, nil
+ return &fleet.AppConfig{Features: fleet.Features{
+ EnableHostUsers: true,
+ EnableSoftwareInventory: true,
+ }}, nil
}
lq.On("QueriesForHost", uint(0)).Return(map[string]string{}, nil)
@@ -3209,7 +3229,8 @@ func TestPolicyWebhooks(t *testing.T) {
ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
return &fleet.AppConfig{
Features: fleet.Features{
- EnableHostUsers: true,
+ EnableHostUsers: true,
+ EnableSoftwareInventory: true,
},
WebhookSettings: fleet.WebhookSettings{
FailingPoliciesWebhook: fleet.FailingPoliciesWebhookSettings{
@@ -3477,7 +3498,10 @@ func TestLiveQueriesFailing(t *testing.T) {
return host, nil
}
ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
- return &fleet.AppConfig{Features: fleet.Features{EnableHostUsers: true}}, nil
+ return &fleet.AppConfig{Features: fleet.Features{
+ EnableHostUsers: true,
+ EnableSoftwareInventory: true,
+ }}, nil
}
ds.PolicyQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) {
return map[string]string{}, nil
@@ -3765,3 +3789,17 @@ func TestPreProcessSoftwareResults(t *testing.T) {
})
}
}
+
+func TestDetailQueriesLinuxDistros(t *testing.T) {
+ for _, linuxPlatform := range fleet.HostLinuxOSs {
+ m := expectedDetailQueriesForPlatform(linuxPlatform)
+ require.Contains(t, m, "users")
+ require.Contains(t, m, "network_interface_unix")
+ require.Contains(t, m, "disk_space_unix")
+ require.Contains(t, m, "os_unix_like")
+ require.Contains(t, m, "orbit_info")
+ require.Contains(t, m, "disk_encryption_linux")
+ require.Contains(t, m, "software_vscode_extensions")
+ require.Contains(t, m, "software_linux")
+ }
+}
diff --git a/server/service/osquery_utils/queries.go b/server/service/osquery_utils/queries.go
index f9a6af2bcd..ecbbf264ca 100644
--- a/server/service/osquery_utils/queries.go
+++ b/server/service/osquery_utils/queries.go
@@ -1055,7 +1055,7 @@ var usersQuery = DetailQuery{
// with many user accounts and groups, this query could be very expensive as the `groups` table
// was generated once for each user.
Query: usersQueryStr,
- Platforms: []string{"linux", "darwin", "windows"},
+ Platforms: append(fleet.HostLinuxOSs, "darwin", "windows"),
DirectIngestFunc: directIngestUsers,
}
diff --git a/server/service/team_policies.go b/server/service/team_policies.go
index 7786c7fe68..75cbe3ae96 100644
--- a/server/service/team_policies.go
+++ b/server/service/team_policies.go
@@ -368,7 +368,8 @@ func (svc *Service) modifyPolicy(ctx context.Context, teamID *uint, id uint, p f
})
}
- var shouldRemoveAll bool
+ var removeAllMemberships bool
+ var removeStats bool
if p.Name != nil {
policy.Name = *p.Name
}
@@ -377,9 +378,8 @@ func (svc *Service) modifyPolicy(ctx context.Context, teamID *uint, id uint, p f
}
if p.Query != nil {
if policy.Query != *p.Query {
- shouldRemoveAll = true
- policy.FailingHostCount = 0
- policy.PassingHostCount = 0
+ removeAllMemberships = true
+ removeStats = true
}
policy.Query = *p.Query
}
@@ -387,6 +387,9 @@ func (svc *Service) modifyPolicy(ctx context.Context, teamID *uint, id uint, p f
policy.Resolution = p.Resolution
}
if p.Platform != nil {
+ if policy.Platform != *p.Platform {
+ removeStats = true
+ }
policy.Platform = *p.Platform
}
if p.Critical != nil {
@@ -395,9 +398,13 @@ func (svc *Service) modifyPolicy(ctx context.Context, teamID *uint, id uint, p f
if p.CalendarEventsEnabled != nil {
policy.CalendarEventsEnabled = *p.CalendarEventsEnabled
}
+ if removeStats {
+ policy.FailingHostCount = 0
+ policy.PassingHostCount = 0
+ }
logging.WithExtras(ctx, "name", policy.Name, "sql", policy.Query)
- err = svc.ds.SavePolicy(ctx, policy, shouldRemoveAll)
+ err = svc.ds.SavePolicy(ctx, policy, removeAllMemberships, removeStats)
if err != nil {
return nil, ctxerr.Wrap(ctx, err, "saving policy")
}
diff --git a/server/service/team_policies_test.go b/server/service/team_policies_test.go
index e6079f1101..9e1a502f67 100644
--- a/server/service/team_policies_test.go
+++ b/server/service/team_policies_test.go
@@ -44,7 +44,7 @@ func TestTeamPoliciesAuth(t *testing.T) {
}
return nil, nil
}
- ds.SavePolicyFunc = func(ctx context.Context, p *fleet.Policy, shouldDeleteAll bool) error {
+ ds.SavePolicyFunc = func(ctx context.Context, p *fleet.Policy, shouldDeleteAll bool, removePolicyStats bool) error {
return nil
}
ds.DeleteTeamPoliciesFunc = func(ctx context.Context, teamID uint, ids []uint) ([]uint, error) {
diff --git a/server/service/testing_utils.go b/server/service/testing_utils.go
index 5b7af86800..82a675ec30 100644
--- a/server/service/testing_utils.go
+++ b/server/service/testing_utils.go
@@ -351,6 +351,7 @@ func RunServerForTestsWithDS(t *testing.T, ds fleet.Datastore, opts ...*TestServ
if len(opts) > 0 {
mdmStorage := opts[0].MDMStorage
scepStorage := opts[0].SCEPStorage
+ commander := apple_mdm.NewMDMAppleCommander(mdmStorage, mdmPusher, cfg.MDM)
if mdmStorage != nil && scepStorage != nil {
err := RegisterAppleMDMProtocolServices(
rootMux,
@@ -358,11 +359,7 @@ func RunServerForTestsWithDS(t *testing.T, ds fleet.Datastore, opts ...*TestServ
mdmStorage,
scepStorage,
logger,
- &MDMAppleCheckinAndCommandService{
- ds: ds,
- commander: apple_mdm.NewMDMAppleCommander(mdmStorage, mdmPusher, cfg.MDM),
- logger: kitlog.NewNopLogger(),
- },
+ NewMDMAppleCheckinAndCommandService(ds, commander, logger),
&MDMAppleDDMService{
ds: ds,
logger: logger,
diff --git a/server/vulnerabilities/nvd/cpe_matching_rule.go b/server/vulnerabilities/nvd/cpe_matching_rule.go
index 222af5e125..32b331d4f3 100644
--- a/server/vulnerabilities/nvd/cpe_matching_rule.go
+++ b/server/vulnerabilities/nvd/cpe_matching_rule.go
@@ -6,7 +6,7 @@ import (
"strings"
"github.com/Masterminds/semver"
- "github.com/facebookincubator/nvdtools/wfn"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/wfn"
)
// CPEMatchingRuleSpec allows you to match against a CPE. Version ranges are supported via SemVer constraints.
diff --git a/server/vulnerabilities/nvd/cpe_matching_rule_test.go b/server/vulnerabilities/nvd/cpe_matching_rule_test.go
index c5f3abc1ca..46f4899ce1 100644
--- a/server/vulnerabilities/nvd/cpe_matching_rule_test.go
+++ b/server/vulnerabilities/nvd/cpe_matching_rule_test.go
@@ -4,7 +4,7 @@ import (
"errors"
"testing"
- "github.com/facebookincubator/nvdtools/wfn"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/wfn"
"github.com/stretchr/testify/require"
)
diff --git a/server/vulnerabilities/nvd/cpe_matching_rules.go b/server/vulnerabilities/nvd/cpe_matching_rules.go
index 76ce97b442..f745f97ec0 100644
--- a/server/vulnerabilities/nvd/cpe_matching_rules.go
+++ b/server/vulnerabilities/nvd/cpe_matching_rules.go
@@ -3,7 +3,7 @@ package nvd
import (
"fmt"
- "github.com/facebookincubator/nvdtools/wfn"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/wfn"
)
type CPEMatchingRules []CPEMatchingRule
diff --git a/server/vulnerabilities/nvd/cpe_test.go b/server/vulnerabilities/nvd/cpe_test.go
index 6f8a5f4c94..bb0b318b18 100644
--- a/server/vulnerabilities/nvd/cpe_test.go
+++ b/server/vulnerabilities/nvd/cpe_test.go
@@ -11,10 +11,10 @@ import (
"testing"
"time"
- "github.com/facebookincubator/nvdtools/cpedict"
"github.com/fleetdm/fleet/v4/pkg/nettest"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/mock"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/cpedict"
kitlog "github.com/go-kit/kit/log"
"github.com/go-kit/log"
"github.com/stretchr/testify/assert"
diff --git a/server/vulnerabilities/nvd/cve.go b/server/vulnerabilities/nvd/cve.go
index ff5661cc14..906c3b7485 100644
--- a/server/vulnerabilities/nvd/cve.go
+++ b/server/vulnerabilities/nvd/cve.go
@@ -15,16 +15,16 @@ import (
"time"
"github.com/Masterminds/semver"
- "github.com/facebookincubator/nvdtools/cvefeed"
- feednvd "github.com/facebookincubator/nvdtools/cvefeed/nvd"
- "github.com/facebookincubator/nvdtools/cvefeed/nvd/schema"
- "github.com/facebookincubator/nvdtools/providers/nvd"
- "github.com/facebookincubator/nvdtools/wfn"
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/ptr"
nvdsync "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/sync"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/cvefeed"
+ feednvd "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/cvefeed/nvd"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/cvefeed/nvd/schema"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/providers/nvd"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/wfn"
"github.com/go-kit/log"
kitlog "github.com/go-kit/log"
"github.com/go-kit/log/level"
diff --git a/server/vulnerabilities/nvd/cve_test.go b/server/vulnerabilities/nvd/cve_test.go
index 8f29cc3e8b..0123d1835d 100644
--- a/server/vulnerabilities/nvd/cve_test.go
+++ b/server/vulnerabilities/nvd/cve_test.go
@@ -11,11 +11,11 @@ import (
"testing"
"time"
- "github.com/facebookincubator/nvdtools/cvefeed"
- "github.com/facebookincubator/nvdtools/wfn"
"github.com/fleetdm/fleet/v4/pkg/nettest"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/mock"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/cvefeed"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/wfn"
"github.com/go-kit/log"
kitlog "github.com/go-kit/log"
"github.com/stretchr/testify/assert"
diff --git a/server/vulnerabilities/nvd/db.go b/server/vulnerabilities/nvd/db.go
index 690132171f..b0e0c33b9e 100644
--- a/server/vulnerabilities/nvd/db.go
+++ b/server/vulnerabilities/nvd/db.go
@@ -6,8 +6,8 @@ import (
"os"
"strings"
- "github.com/facebookincubator/nvdtools/cpedict"
- "github.com/facebookincubator/nvdtools/wfn"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/cpedict"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/wfn"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3"
)
diff --git a/server/vulnerabilities/nvd/indexed_cpe_item.go b/server/vulnerabilities/nvd/indexed_cpe_item.go
index 3bc3415027..f2e38086d2 100644
--- a/server/vulnerabilities/nvd/indexed_cpe_item.go
+++ b/server/vulnerabilities/nvd/indexed_cpe_item.go
@@ -1,8 +1,8 @@
package nvd
import (
- "github.com/facebookincubator/nvdtools/wfn"
"github.com/fleetdm/fleet/v4/server/fleet"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/wfn"
)
type IndexedCPEItem struct {
diff --git a/server/vulnerabilities/nvd/sync.go b/server/vulnerabilities/nvd/sync.go
index 6426303897..a4405b7c9d 100644
--- a/server/vulnerabilities/nvd/sync.go
+++ b/server/vulnerabilities/nvd/sync.go
@@ -15,12 +15,12 @@ import (
"github.com/fleetdm/fleet/v4/server/contexts/license"
- "github.com/facebookincubator/nvdtools/cvefeed"
- feednvd "github.com/facebookincubator/nvdtools/cvefeed/nvd"
"github.com/fleetdm/fleet/v4/pkg/download"
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/ptr"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/cvefeed"
+ feednvd "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/cvefeed/nvd"
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
)
diff --git a/server/vulnerabilities/nvd/sync/cve_syncer.go b/server/vulnerabilities/nvd/sync/cve_syncer.go
index 065d8faec8..178429cf65 100644
--- a/server/vulnerabilities/nvd/sync/cve_syncer.go
+++ b/server/vulnerabilities/nvd/sync/cve_syncer.go
@@ -21,11 +21,11 @@ import (
"strings"
"time"
- "github.com/facebookincubator/nvdtools/cvefeed/nvd/schema"
"github.com/fleetdm/fleet/v4/orbit/pkg/constant"
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
"github.com/fleetdm/fleet/v4/server/ptr"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/cvefeed/nvd/schema"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/pandatix/nvdapi/common"
@@ -36,7 +36,7 @@ import (
// to the directory specified in the dbDir field in the form of JSON files.
// It stores the CVE information using the legacy feed format.
// The reason we decided to store in the legacy format is because
-// the github.com/facebookincubator/nvdtools doesn't yet support parsing
+// the github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools doesn't yet support parsing
// the new API 2.0 JSON format.
type CVE struct {
client *http.Client
@@ -183,7 +183,7 @@ func (s *CVE) update(ctx context.Context) error {
func (s *CVE) updateYearFile(year int, cves []nvdapi.CVEItem) error {
// The NVD legacy feed files start at year 2002.
- // This is assumed by the facebookincubator/nvdtools package.
+ // This is assumed by the github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools package.
if year < 2002 {
year = 2002
}
diff --git a/server/vulnerabilities/nvd/sync/cve_syncer_test.go b/server/vulnerabilities/nvd/sync/cve_syncer_test.go
index d58e7449c4..b25e0fbefc 100644
--- a/server/vulnerabilities/nvd/sync/cve_syncer_test.go
+++ b/server/vulnerabilities/nvd/sync/cve_syncer_test.go
@@ -18,7 +18,7 @@ import (
"testing"
"time"
- "github.com/facebookincubator/nvdtools/cvefeed/nvd/schema"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/cvefeed/nvd/schema"
"github.com/go-kit/log"
"github.com/google/go-cmp/cmp"
"github.com/pandatix/nvdapi/v2"
diff --git a/server/vulnerabilities/nvd/tools/HOWTO.md b/server/vulnerabilities/nvd/tools/HOWTO.md
new file mode 100644
index 0000000000..937f12d565
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/HOWTO.md
@@ -0,0 +1,335 @@
+# How to use nvdtools
+
+The command line tools provided by nvdtools were designed for processing inventory data in pipelines.
+
+To start, you will need a vulnerability database. In this toolkit you'll find the [nvdsync](https://github.com/facebookincubator/nvdtools/tree/master/cmd/nvdsync) command, which can download the public NVD database to local disk:
+
+```bash
+nvdsync -v=1 -cve_feed=cve-1.0.json.gz /tmp/nvd
+```
+
+Next up, you need a data collector to create a CPE inventory. Collectors are domain-specific programs capable of acquiring asset information (e.g. a list of hardware, or packages in a repo or system) and printing this information to standard output.
+
+Think of the simplest data collector as an execution of rpm (or repoquery):
+
+```bash
+rpm -qa | rpm2cpe -rpm=1 -cpe=2
+```
+
+This collector [rpm2cpe](https://github.com/facebookincubator/nvdtools/tree/master/cmd/rpm2cpe) will use the name of the rpm files in column 1 of the input, produce a CPE in column 2, and print both to standard output.
+
+Finally, use the [cpe2cve](https://github.com/facebookincubator/nvdtools/tree/master/cmd/cpe2cve) processor to consume the CPE inventory from standard input and print CVEs affecting which CPEs to standard output:
+
+```bash
+rpm -qa | \
+rpm2cpe -rpm=1 -cpe=2 | \
+cpe2cve -cpe=2 -cve=3 -cwe=4 /tmp/nvd/*.json.gz
+```
+
+The command above process each CPE individually and prints their respective CVEs. However, it's not uncommon in the NVD database to have more elaborate CVEs which affect a combination of CPEs, e.g. if A and B and not C. For this case, you could group your CPEs per host, for example, and process them in a single batch:
+
+```bash
+set -o pipefail
+(hostname
+rpm -qa | rpm2cpe -rpm=1 -cpe=2 -e=1 | sort -u | paste -s -d, | \
+cpe2cve -cpe=1 -cve=2 -e=1 /tmp/nvd/*.json.gz | paste -s -d,) | paste -s -d'\t'
+```
+
+The command above prints a single line containing ` ` for your machine. Great, but is unrealistic to use in each machine in production systems. That's when things start to get more interesting. See the next section for how to decouple this pipeline from collection to processing and reporting.
+
+# Using nvdtools in production
+
+In order to effectively use nvdtools, you will likely want to decouple data collection from processing and reporting.
+
+The idea is to use nvdtools as the building blocks of a much larger system that orchestrates data collection separately from processing, leaving the processing and reporting (heavy lifting) to be executed in a data warehouse.
+
+Starting from the data collection, think of the different inventory classes that may exist in the environment:
+
+* Hosted software: packages sitting in software repositories, available to your fleet (source and binary, first-party and third-party)
+* Installed software: packages installed on machines or containers, ideally from your managed repositories
+* Running software: processes executing on machines or containers, ideally from a known package
+* Hardware: a list of hardware parts that can be used to create CPEs, e.g. `cpe:/h:dell:inspiron:8500`
+
+The collecting stage have different requirements for each class. The processing stage consume inventories from these collector classes and process them with specialized vulnerability databases.
+
+The public NVD database covers a great deal of open source software and common hardware. However, there are several ecosystems that may be present in your infrastructure (php, python, nodeJS, go) but not well covered by the NVD database alone.
+
+To maximize vulnerability matching and coverage (and data quality, later user experience on reports), consider using multiple database providers. You will need to convert their databases to the [NVD CVE JSON 1.0](https://csrc.nist.gov/schema/nvd/feed/1.0/nvd_cve_feed_json_1.0.schema) format to use them with the cpe2cve processor.
+
+Once the data from collectors is decoupled from processing, the nvdtools can be used to process large inventories with millions of assets more efficiently.
+
+The following sections cover collectors and processors in a bit more depth.
+
+## Collectors
+
+This section covers some of the inventory classes mentioned above.
+
+Collectors are all about retrieving asset information and providing enough data to build CPEs for late processing.
+
+### Hosted software collectors
+
+These are domain-specific programs that scrape software repositories (or logs) and report packages available to the fleet.
+
+Examples of hosted software collectors are programs to report packages hosted in yum, maven, munki, chocolatey, docker registry.
+
+Data provided by hosted software collectors must contain enough information to create CPEs, comprising at least the asset type (part; a=sw, h=hw, o=os), product and version. Other fields like vendor and target hardware can improve vulnerability matching later, but are not blockers to get started.
+
+Orchestrating the execution of collectors is platform dependent. At the very least, a cron-like system could periodically run collectors and store their data in files and/or a database.
+
+Here's an example of cron-like job to scrape all yum repositories configured on the machine running the collector:
+
+```bash
+Q=('{"vendor":"%{VENDOR}","product":"%{NAME}","version":"%{VERSION}","update":"%{RELEASE}","target_hw":"%{ARCH}","metadata":{"product_group":"%{REPO}","package_name":"%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}.rpm","package_source":"%{SOURCERPM}"}}')
+
+set -o pipefail
+repoquery -C --all --queryformat "${Q[@]}" | \
+jq -r '[ "a", .vendor, .product, .version // "-", .update, .sw_edition, .target_sw, .target_hw, ( .metadata | tojson ) ] | @csv' | \
+csv2cpe \
+ -cpe_part=1 \
+ -cpe_vendor=2 \
+ -cpe_product=3 \
+ -cpe_version=4 \
+ -cpe_update=5 \
+ -cpe_swedition=6 \
+ -cpe_targetsw=7 \
+ -cpe_targethw=8 \
+ -e=1 \
+ -i=1 \
+ -lower \
+ -o=$'\t'
+```
+
+The `-e=1` flag erases the injected "a" part from jq, and the `-i=1` flag tells [csv2cpe](https://github.com/facebookincubator/nvdtools/tree/master/cmd/csv2cpe) to add the cpe in column 1 of its output.
+
+The tab-separated output contains the following columns:
+
+```
+cpe, vendor, product, version, update, sw_edition, target_sw, target_hw, metadata_json
+```
+
+This type of output can be stored in a database such as MySQL by simply adding `mysqlimport` at the end of the pipeline; or write the output to a message queue in similar fashion.
+
+If executing jq and csv2cpe along with the collector is not an option, you can always store the raw JSON inventory and later execute jq and csv2cpe in the processing stage of the pipeline.
+
+Notice the metadata field: that information may be helpful much later on the processing and reporting stages, allowing your system to report packages in such a way that your users understand them, avoiding people having to learn the CPE format and details of your system.
+
+### Installed software collectors
+
+There are several ways of collecting information about packages installed on a system. We mostly use [osquery](https://osquery.io/) for this, taking periodic snapshots of what is installed on a machine and shipping the data to the data warehouse.
+
+The main advantage of using osquery is to support all major operating systems with a SQL-like interface for collecting information.
+
+The osquery results are used to build CPEs which are later processed in batches.
+
+Here's an example of a query to collect the macOS operating system version and all apps installed:
+
+```bash
+Q=("
+SELECT
+ 'o' AS part,
+ 'apple' AS vendor,
+ os.name AS product,
+ os.version
+FROM
+ os_version AS os
+;
+SELECT
+ 'a' AS part,
+ '' AS vendor,
+ bundle_name AS product,
+ bundle_version AS version
+FROM
+ apps
+WHERE
+ bundle_name IS NOT NULL AND bundle_name <> ''
+;
+")
+
+osqueryi --json "${Q[@]}"
+```
+
+Although osquery supports a `--csv` flag, the JSON output gives flexibility (e.g. handling NULL values) and we can use jq to re-format to CSV, then use csv2cpe to produce the installed software inventory:
+
+```bash
+set -o pipefail
+osqueryi --json "${Q[@]}" | \
+jq -r '.[] | [.part, .vendor, .product, .version // "-"] | @csv' | \
+csv2cpe \
+ -cpe_part=1 \
+ -cpe_vendor=2 \
+ -cpe_product=3 \
+ -cpe_version=4 \
+ -e=1 \
+ -i=1 \
+ -lower \
+ -o=$'\t'
+```
+
+Converting NULL versions to '-' tells the processor (much later, when cpe2cve is run) to handle dash as "Not Available" during CVE matching, instead of "Any" for empty space.
+
+RPM packages have richer information, and provide extra metadata for matching CPEs against data from the hosted software collector. Following is a more complex query returning host RPM inventory with metadata:
+
+```bash
+Q=("
+SELECT
+ 'o' AS part,
+ 'centos' AS vendor,
+ 'centos' AS product,
+ (os.major || '.' || os.minor || '.' || os.patch) AS version,
+ '' AS release,
+ sys.cpu_type AS target_hw,
+ NULL as metadata
+FROM
+ os_version AS os, system_info AS sys
+;
+SELECT
+ 'a' AS part,
+ '' AS vendor,
+ name AS product,
+ version,
+ release,
+ arch AS target_hw,
+ JSON_OBJECT(
+ 'package_name', (name || '-' || version || '-' || release || '.' || arch || '.rpm'),
+ 'package_source', source,
+ 'package_sha1', sha1,
+ 'package_size', size
+ ) AS metadata
+FROM
+ rpm_packages
+;
+")
+
+set -o pipefail
+osqueryi --json "${Q[@]}" | \
+jq -r '.[] | [.part, .vendor, .product, .version // "-", .release, .target_hw, .metadata] | @csv' | \
+csv2cpe \
+ -cpe_part=1 \
+ -cpe_vendor=2 \
+ -cpe_product=3 \
+ -cpe_version=4 \
+ -cpe_update=5 \
+ -cpe_targethw=6 \
+ -e=1 \
+ -i=1 \
+ -lower \
+ -o=$'\t'
+```
+
+Similarly to the hosted software collectors, it's up to you to ship raw osquery JSON to a database or message queue, and execute jq and csv2cpe in the processing stage of the pipeline. Also, you'll likely want to record the hostname where the query was executed. Check out the system_info osquery table for details.
+
+### Running software collectors
+
+Process information alone is not very useful for vulnerability scanning. Moreover, you have to choose between collecting samples (a snapshot of ps) or hook up into the OS to track all process executions.
+
+This data is expensive to collect, decorate (enrich with useful information), and move around - can be massive in size. Ask yourself whether this is really needed in your environment.
+
+Nonetheless, following query is an example for osquery that can capture process information, reporting the RPM package where the binary comes from, along with process-related metadata.
+
+Note: this query can take a few minutes to run depending on how many processes and packages your system have.
+
+```bash
+Q=("
+SELECT
+ 'a' AS part,
+ '' AS vendor,
+ pkg.name AS product,
+ pkg.version,
+ pkg.release,
+ pkg.arch AS target_hw,
+ JSON_OBJECT(
+ 'package_name', (pkg.name || '-' || pkg.version || '-' || pkg.release || '.' || pkg.arch || '.rpm'),
+ 'package_source', pkg.source,
+ 'package_sha1', pkg.sha1,
+ 'package_size', pkg.size,
+ 'process_name', proc.name,
+ 'process_parent', proc.parent,
+ 'process_cwd', proc.cwd,
+ 'process_cmd', proc.cmdline,
+ 'process_pid', proc.pid,
+ 'process_start_time', proc.start_time
+ ) AS metadata
+FROM (
+ SELECT * FROM processes WHERE path <> ''
+) AS proc
+JOIN (
+ SELECT * FROM rpm_package_files
+ WHERE package <> '' AND path <> ''
+) AS pkg_files
+ON
+ proc.path = pkg_files.path
+JOIN (
+ SELECT * FROM rpm_packages WHERE name <> ''
+) AS pkg
+ON
+ pkg_files.package = pkg.name
+")
+
+set -o pipefail
+osqueryi --json "${Q[@]}" | \
+jq -r '.[] | [.part, .vendor, .product, .version // "-", .release, .target_hw, .metadata] | @csv' | \
+csv2cpe \
+ -cpe_part=1 \
+ -cpe_vendor=2 \
+ -cpe_product=3 \
+ -cpe_version=4 \
+ -cpe_update=5 \
+ -cpe_targethw=6 \
+ -e=1 \
+ -i=1 \
+ -lower \
+ -o=$'\t'
+```
+
+osquery also supports collecting information from docker containers, their networks, and images. This can be useful if you have an inventory of images in a managed registry.
+
+Metadata can be used later to join against data from the hosted and/or installed software collectors.
+
+## Processors
+
+The main processor covered in this section is cpe2cve, the vulnerability matching processor.
+
+Once collectors are producing data and CPEs are available (or can be built), the cpe2cve processor can perform CVEs matching and produce reports. The output of cpe2cve is always one CVE per line, regardless of whether the input was a single CPE or a group or CPEs.
+
+Given the different inventories, you may want different vulnerability databases to process them. As previously mentioned, the public NVD database alone is generally not enough for good coverage. Specialized ecosystems (e.g. nodejs, ruby, python, php, go) require specialized databases.
+
+### Vulnerability Databases
+
+It is recommended to use multi-vendor databases. The cpe2cve processor require databases in the NVD CVE JSON 1.0 format, as files on disk. XML is also supported but discouraged, and likely to be deprecated - XML databases don't support the concept of version ranges, resulting in lower quality CVE matching and reporting.
+
+On a system with multi-vendor databases, the maintainers of collectors should be able to define which database(s) to use to process their inventory. For example, the yum collector maintainer would pick the NVD database, but the nodejs collector maintainer would prefer a specialized database, e.g. snyk.
+
+### Vulnerability Database: patches, snoozes, edits
+
+It's not uncommon for processors to report false positives due to the quality of the inventory and databases, lack of normalization (missing vendors, wrong product names, bad versions).
+
+Curating the data is the hardest part of maintaining a large system with multiple inventories and databases. Reporting high quality data is generally what makes the system successful.
+
+Following are some methods that can help improve the data quality and end-user experience:
+
+* Patch reports: allow the collectors to report patches applied to their source code; this can avoid reporting false positives by effectively `grep -v`'ing a list of patched CVEs from the processor output
+* Snoozes: let users snooze certain CVEs, in the sense of not reporting them for a period of time or indefinitely; this can avoid reporting false positives consecutively
+* Edits: some times the quality of the vulnerability database is subpar, missing information, or containing incorrect information; allowing edits to existing CVEs or creating new CVEs can improve the quality of matching and reports
+
+### The [cpe2cve](https://github.com/facebookincubator/nvdtools/blob/master/cmd/cpe2cve) vulnerability processor
+
+Using the cpe2cve vulnerability processor is pretty straightforward, but it's worth highlighting a few things:
+
+* The quality of the vulnerability matching (CVE) results depend entirely on the quality of the CPE inventory and the vulnerability database being used (garbage in -> garbage out)
+* Using specialized vulnerability databases for specific inventories can increase the quality of the results
+* Processing one CPE alone may not yield all vulnerabilities; CVE databases use conditional logic (expressions) to match CPEs: if A and B or C
+* Processing inventories from hosted software collectors generally process each CPE individually and does not account for their dependencies; putting that data together is not part of nvdtools
+* Processing inventories from installed or running software collectors yield better results when all CPEs are grouped and processed in one batch; ideally with an operating system CPE (cpe:/o) in addition to all packages (cpe:/a)
+* Consider whether you really need to process inventories from installed and running software using cpe2cve: this can be expensive depending on the size of your fleet; you may want to start by simply matching CPEs back to your hosted software inventories
+* Use patching information, snoozes, and edits on top of the CVE database - in pre or post processing stages - to avoid false positives and consecutive false positives resulting in anger and disappointment from your users
+* Use the metadata from collectors to build high quality reports for their maintainers, present data that know about (their own package names, not CPEs)
+
+All the collector examples in previous sections put their generated CPE (or comma-separated list of CPEs) in the first column of their output. Their output has tab-separated columns. Those are also the default delimiters for the cpe2cve input (check --help).
+
+With the examples above, an execution of the CVE processor could take CPE(s) from column 1 of the input, and insert CVE in the same column, pushing the original input one column forward:
+
+```bash
+cat inventory.csv | cpe2cve -cpe=1 -cve=1 /tmp/nvd/*.json.gz
+```
+
+Check out the [--help](https://github.com/facebookincubator/nvdtools/blob/master/cmd/cpe2cve/cpe2cve.go#L51) flag for all options related to input and output delimiters, lists, caching, and extra columns you may want to add to the output, such as CVSS score and CWE of each CVE.
diff --git a/server/vulnerabilities/nvd/tools/LICENSE b/server/vulnerabilities/nvd/tools/LICENSE
new file mode 100644
index 0000000000..261eeb9e9f
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/server/vulnerabilities/nvd/tools/Makefile b/server/vulnerabilities/nvd/tools/Makefile
new file mode 100644
index 0000000000..75a5e0dc06
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/Makefile
@@ -0,0 +1,157 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+NAME = nvdtools
+VERSION = tip
+
+TOOLS = \
+ cpe2cve \
+ csv2cpe \
+ fireeye2nvd \
+ flexera2nvd \
+ idefense2nvd \
+ nvdsync \
+ rpm2cpe \
+ rustsec2nvd \
+ snyk2nvd \
+ vulndb
+
+DOCS = \
+ CODE_OF_CONDUCT.md \
+ CONTRIBUTING.md \
+ HOWTO.md \
+ LICENSE \
+ README.md
+
+GO = go
+GOOS = $(shell $(GO) env GOOS)
+GOARCH = $(shell $(GO) env GOARCH)
+
+TAR = tar
+ZIP = zip
+INSTALL = install
+
+# Compile all tools.
+all: $(TOOLS)
+
+# Compile TOOLS to ./build/bin/$tool using GOOS and GOARCH.
+$(TOOLS):
+ GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO) build $(GOFLAGS) -o ./build/bin/$@ ./cmd/$@
+
+# Check/fetch all dependencies.
+deps:
+ GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO) get -v -d ./...
+
+# install installs tools and documentation.
+# The install target is used by rpm and deb builders.
+install:
+ # tools
+ $(INSTALL) -d $(DESTDIR)/usr/bin
+ for tool in $(TOOLS); do $(INSTALL) -p -m 0755 ./build/bin/$$tool $(DESTDIR)/usr/bin/$$tool; done
+ # docs
+ $(INSTALL) -d $(DESTDIR)/usr/share/doc/nvdtools
+ for doc in $(DOCS); do $(INSTALL) -p -m 0644 $$doc $(DESTDIR)/usr/share/doc/nvdtools/$$doc; done
+
+DIST_NAME = $(NAME)-$(VERSION)
+DIST_DIR = build/$(DIST_NAME)
+
+# binary_dist creates a local binary distribution in DIST_DIR.
+binary_dist: $(TOOLS)
+ mkdir -p $(DIST_DIR)/doc
+ cp $(DOCS) $(DIST_DIR)/doc
+ mv build/bin $(DIST_DIR)/bin
+
+# binary_tar creates tarball of binary distribution.
+binary_tar: binary_dist
+ mkdir -p build/tgz
+ cd build && $(TAR) czf tgz/$(DIST_NAME)-$(GOOS)-$(GOARCH).tar.gz $(DIST_NAME)
+ rm -rf $(DIST_DIR)
+
+# binary_zip creates zip of binary distribution.
+binary_zip: binary_dist
+ mkdir -p build/zip
+ cd build && $(ZIP) -r zip/$(DIST_NAME)-$(GOOS)-$(GOARCH).zip $(DIST_NAME)
+ rm -rf $(DIST_DIR)
+
+# binary_deb creates debian package.
+#
+# Requires GOPATH and dependencies available to compile nvdtools.
+# Must set version to build: make binary_deb VERSION=1.0
+binary_deb:
+ VERSION=$(VERSION) dpkg-buildpackage -rfakeroot -uc -us
+ mkdir -p build/deb
+ mv ../$(NAME)*.deb build/deb/
+
+# archive_tar creates tarball of the source code.
+archive_tar:
+ mkdir -p build/tgz
+ $(TAR) czf build/tgz/$(DIST_NAME).tar.gz \
+ --exclude=build \
+ --exclude=release \
+ --exclude=.git \
+ --exclude=.travis.yml \
+ --transform s/./$(DIST_NAME)/ \
+ .
+
+# binary_rpm creates rpm package.
+#
+# Requires GOPATH and dependencies available to compile nvdtools.
+# Must set version to build: make binary_rpm VERSION=1.0
+binary_rpm: archive_tar
+ mkdir -p build/rpm/SOURCES
+ mv build/tgz/$(DIST_NAME).tar.gz build/rpm/SOURCES/
+ rpmbuild -ba \
+ --define="_topdir $(PWD)/build/rpm" \
+ --define="_version $(VERSION)" \
+ nvdtools.spec
+
+# release_tar creates tarball releases.
+release_tar:
+ mkdir -p release
+ make deps binary_tar GOOS=darwin GOARCH=amd64
+ make deps binary_tar GOOS=freebsd GOARCH=amd64
+ make deps binary_tar GOOS=freebsd GOARCH=arm
+ make deps binary_tar GOOS=linux GOARCH=amd64
+ make deps binary_tar GOOS=linux GOARCH=arm64
+ mv build/tgz/*.tar.gz release
+
+# release_zip creates zip releases.
+release_zip:
+ mkdir -p release
+ make deps binary_zip GOOS=windows GOARCH=386
+ make deps binary_zip GOOS=windows GOARCH=amd64
+ mv build/zip/*.zip release
+
+# release_deb creates debian releases.
+release_deb: binary_deb
+ mkdir -p release
+ mv build/deb/*.deb release
+
+# release_rpm creates rpm releases.
+release_rpm: binary_rpm
+ mkdir -p release
+ mv build/rpm/RPMS/*/*.rpm release
+
+# release creates all release packages.
+# Example: make distclean release VERSION=1.0
+release: release_deb release_rpm release_tar release_zip
+
+# Removes build related files.
+clean:
+ rm -rf build
+
+distclean: clean
+ rm -rf release
+
+.PHONY: $(TOOLS)
diff --git a/server/vulnerabilities/nvd/tools/README.md b/server/vulnerabilities/nvd/tools/README.md
new file mode 100644
index 0000000000..d5685d52c0
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/README.md
@@ -0,0 +1,217 @@
+> The contents of this directory were copied (in April 2024) from https://github.com/facebookincubator/nvdtools.git.
+---
+
+
+
+# NVD Tools
+
+A collection of tools for working with [National Vulnerability Database](https://nvd.nist.gov/) feeds.
+
+The [HOWTO](HOWTO.md) provides a broader view on how to effectively use these tools.
+
+---
+
+* [Requirements](#requirements)
+* [Installation](#installation)
+* [How build](#How-build)
+* [Command line tools](#command-line-tools)
+ * [cpe2cve](#cpe2cve)
+ * [csv2cpe](#cpe2cve)
+ * [fireeye2nvd](#fireeye2nvd)
+ * [flexera2nvd](#flexera2nvd)
+ * [idefense2nvd](#idefense2nvd)
+ * [nvdsync](#nvdsync)
+ * [rpm2cpe](#rpm2cpe)
+ * [rustsec2nvd](#rustsec2nvd)
+ * [vfeed2nvd](#vfeed2nvd)
+ * [vulndb](#vulndb)
+* [Libraries](#libraries)
+ * [cvss2](#cvss2)
+ * [cvss3](#cvss3)
+ * [wfn](#wfn)
+* [License](#license)
+
+---
+
+## Requirements
+
+* Go 1.13 or newer
+
+## Installation
+
+You need a properly setup Go environment.
+
+#### Download and install NVD Tools:
+
+For Go 1.13 - 1.14:
+```bash
+go get github.com/facebookincubator/nvdtools/...
+cd "$GOPATH"/src/github.com/facebookincubator/nvdtools/cmd
+go install ./...
+```
+
+From Go 1.15 onwards, modules are not downloaded to `GOPATH`, but to `GOMODCACHE`. It is recommended to clone the repo and run run go install from there instead:
+```bash
+git clone https://github.com/facebookincubator/nvdtools
+cd nvdtools
+go install ./...
+```
+
+From Go 1.17 onwards, `go get` is deprecated. `go install` is used instead to download the module to the cache and install it:
+```bash
+go install github.com/facebookincubator/nvdtools/...@latest
+```
+
+## How-build
+```bash
+go mod init github.com/facebookincubator/nvdtools
+go mod tidy
+make
+cp build/bin/* ~/go/bin/
+
+```
+
+## Command line tools
+
+### `cpe2cve`
+
+*cpe2cve* is a command line tool for scanning an inventory of CPE names for vulnerabilities.
+
+It expects a stream of lines of delimiter-separated fields, one of these fields being a delimiter-separated list of CPE names in the inventory.
+
+Vulnerability feeds should be provided as arguments to the program in JSON format.
+
+Output is a stream of delimiter-separated input value decorated with a vulnerability ID (CVE) and a delimiter-separated list of CPE names that match this vulnerability.
+
+Unwanted input fields could be erased from the output with `-e` option.
+
+Input and output delimiters can be configured with `-d`, `-d2`, `-o` an `-o2` options.
+
+The column to which output the CVE and matches for that CVE can be configured with `-cve` and `-matches` options correspondingly.
+
+### download data
+```bash
+curl -o- -s -k -v https://nvd.nist.gov/vuln/data-feeds >data-feeds.html
+cat data-feeds.html|grep -Eo '(/feeds\/[^"]*\.gz)'|xargs -I % wget -c https://nvd.nist.gov%
+```
+
+#### Example 1: scan a software for vulnerabilities
+
+```bash
+echo "cpe:/a:apache"|cpe2cve -cpe 1 -e 1 -cve 1 nvdcve-1.1-*.json.gz
+echo "cpe:/a:gnu:glibc:2.28" | cpe2cve -cpe 1 -e 1 -cve 1 nvdcve-1.0-*.json.gz
+CVE-2009-4881
+CVE-2015-8985
+CVE-2016-4429
+CVE-2010-3192
+CVE-2010-4756
+```
+
+#### Example 2: find vulnerabilities in software inventory per production host
+
+```bash
+./cpe2cve -d ' ' -d2 , -o ' ' -o2 , -cpe 2 -e 2 -matches 3 -cve 2 nvdcve-1.0-*.json.gz << EOF
+host1.foo.bar cpe:/a:gnu:glibc:2.28,cpe:/a:gnu:zlib:1.2.8
+host2.foo.bar cpe:/a:gnu:glibc:2.28,cpe:/a:haxx:curl:7.55.0
+EOF
+host1.foo.bar CVE-2009-4881 cpe:/a:gnu:glibc:2.28
+host1.foo.bar CVE-2016-4429 cpe:/a:gnu:glibc:2.28
+host2.foo.bar CVE-2014-5119 cpe:/a:gnu:glibc:2.28
+host2.foo.bar CVE-2016-4429 cpe:/a:gnu:glibc:2.28
+host2.foo.bar CVE-2018-1000120 cpe:/a:haxx:curl:7.55.0
+host2.foo.bar CVE-2018-1000122 cpe:/a:haxx:curl:7.55.0
+host2.foo.bar CVE-2010-4756 cpe:/a:gnu:glibc:2.28
+host2.foo.bar CVE-2017-8817 cpe:/a:haxx:curl:7.55.0
+```
+
+### `csv2cpe`
+
+*csv2cpe* is a tool that generates an URI-bound CPE from CSV input, flags configure the meaning of each input field:
+
+* `-cpe_part` -- identifies the class of a product: h for hardware, a for application and o for OS
+* `-cpe_vendor` -- identifies the person or organisation that manufactured or created the product
+* `-cpe_product` -- describes or identifies the most common and recognisable title or name of the product
+* `-cpe_version` -- vendor-specific alphanumeric strings characterising the particular release version of the product
+* `-cpe_update` -- vendor-specific alphanumeric strings characterising the particular update, service pack, or point release of the product
+* `-cpe_edition` -- capture edition-related terms applied by the vendor to the product; this attribute is considered deprecated in CPE specification version 2.3 and it should be assigned the logical value ANY except where required for backward compatibility with version 2.2 of the CPE specification.
+* `-cpe_swedition` -- characterises how the product is tailored to a particular market or class of end users
+* `-cpe_targetsw` -- characterises the software computing environment within which the product operates
+* `-cpe_targethw` -- characterises the software computing environment within which the product operates
+* `-cpe_language` -- defines the language supported in the user interface of the product being described; must be valid language tags as defined by [RFC5646]
+* `-cpe_other` -- any other general descriptive or identifying information which is vendor- or product-specific and which does not logically fit in any other attribute value
+
+Omitted parts of the CPE name defaults to logical value ANY, as per [specification](https://nvlpubs.nist.gov/nistpubs/Legacy/IR/nistir7695.pdf)
+
+Optional flag `-lower` brings the strings to lower case.
+
+#### Example: generate URI-bound CPE name out of comma-separated list of attributes
+
+```bash
+$ echo 'a,Microsoft,Internet Explorer,8.1,SP1,-,*' | csv2cpe -x -lower -cpe_part=1 -cpe_vendor=2 -cpe_product=3 -cpe_version=4 -cpe_update=5 -cpe_edition=6 -cpe_language=7
+cpe:/a:microsoft:internet_explorer:8.1:sp1:-
+```
+
+### `fireeye2nvd`
+
+*fireeye2nvd* downloads the vulnerability data from [FireEye](https://www.fireeye.com/) and converts it into NVD format. The resulting file can be used as a feed in [`cpe2cve`](#cpe2cve) processor
+
+### `flexera2nvd`
+
+*flexera2nvd* downloads the vulnerability data from [Flexera](https://www.flexera.com/) and converts it into NVD format. The resulting file can be used as a feed in [`cpe2cve`](#cpe2cve) processor
+
+### `idefense2nvd`
+
+*idefense2nvd* downloads the vulnerability data from Idefense and converts it into NVD format. The resulting file can be used as a feed in [`cpe2cve`](#cpe2cve) processor
+
+### `nvdsync`
+
+*nvdsync* synchronizes NVD data feeds to local directory; it checks the hashes of the files against the ones provided by NVD and only updates the changed files.
+
+### `rpm2cpe`
+
+*rpm2cpe* takes a delimiter-separated input with one of the fields containing RPM package name and produces delimiter-separated output consisting of the same fields plus CPE name parsed from RPM package name.
+
+#### Example: generate URI-bound CPE name out of RPM package filename
+
+```bash
+echo openoffice-eu-writer-4.1.5-9789.i586.rpm | rpm2cpe -rpm=1 -cpe=2 -e=1
+cpe:/a::openoffice-eu-writer:4.1.5:9789:~~~~i586~
+```
+
+### `rustsec2nvd`
+
+*rustsec2nvd* converts the vulnerabilities from the [Rustsec Advisory-DB](https://github.com/RustSec/advisory-db) into NVD format. The resulting file can be used as a feed in [`cpe2cve`](#cpe2cve) processor
+
+### `snyk2nvd`
+
+*snyk2nvd* downloads the vulnerability data from [Snyk](https://snyk.io/) and converts it into NVD format. The resulting file can be used as a feed in [`cpe2cve`](#cpe2cve) processor
+
+### `vfeed2nvd`
+
+*vfeed2nvd* converts the vulnerability data from [vFeed](https://vfeed.io/) into NVD format. The resulting file can be used as a feed in [`cpe2cve`](#cpe2cve) processor
+
+### `vulndb`
+
+*vulndb* is a command line tool to manage NVD-like vulnerability databases, backed by MySQL.
+
+Supports NVD CVE JSON 1.0 feeds. Data is versioned, organized by provider names and grouped by vendor, custom, and snoozes datasets:
+
+* Vendor dataset: read-only CVE feeds we continuously import.
+* Custom dataset: allows to overwrite CVEs from vendor data with custom data during exports
+* Snooze dataset: user-defined CVE and metadata with deadline, used for remediation automation
+
+See `vulndb help` for details.
+
+## Libraries
+
+### cvss2
+
+Implementation of [CVSS v2 specification](https://www.first.org/cvss/v2/guide) which provides functions for serializing and deserializing vectors as well as score calculation.
+
+### cvss3
+
+Implementation of [CVSS v3 specification](https://www.first.org/cvss/specification-document) which provides functions for serializing and deserializing vectors as well as score calculation.
+
+## License
+
+nvdtools licensed under Apache License, Version 2.0, as found in the [LICENSE](LICENSE) file.
diff --git a/server/vulnerabilities/nvd/tools/cpedict/cpedict.go b/server/vulnerabilities/nvd/tools/cpedict/cpedict.go
new file mode 100644
index 0000000000..3bb45eee4e
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cpedict/cpedict.go
@@ -0,0 +1,139 @@
+// Package cpedict defines the types and methods necessary to parse and lookup CPE dictionary conforming to
+// CPE Dictionary specification 2.3 as per https://nvlpubs.nist.gov/nistpubs/Legacy/IR/nistir7697.pdf.
+// The implementation is not full, only parts required to parse NVD vulnerability feed are implemented
+//
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package cpedict
+
+import (
+ "encoding/xml"
+ "io"
+ "time"
+
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/wfn"
+)
+
+// TextType represents multi-language text
+type TextType map[string]string
+
+// UnmarshalXML -- load TextType from XML
+func (t *TextType) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+ var text string
+ lang := "en"
+ if *t == nil {
+ *t = TextType{}
+ }
+ for _, attr := range start.Attr {
+ if attr.Name.Local == "lang" {
+ lang = attr.Value
+ }
+ }
+ if err := d.DecodeElement(&text, &start); err != nil {
+ return err
+ }
+ (*t)[lang] = text
+ return nil
+}
+
+// PlatformType -- NVD doesn't use it
+// TODO: implement
+// type PlatformType struct{}
+
+// CheckFactRefType is a reference to a check that always evaluates to
+// TRUE, FALSE, or ERROR. Examples of types of checks are OVAL and OCIL checks.
+// NVD doesn't use it
+// TODO: implement
+// type CheckFactRefType struct{}
+
+// NamePattern represents CPE name
+type NamePattern wfn.Attributes
+
+// UnmarshalXMLAttr implements xml.UnmarshalerAttr interface
+func (np *NamePattern) UnmarshalXMLAttr(attr xml.Attr) error {
+ wfn, err := wfn.Parse(attr.Value)
+ if err != nil {
+ return err
+ }
+ *np = (NamePattern)(*wfn)
+ return nil
+}
+
+func (np NamePattern) String() string {
+ return wfn.Attributes(np).String()
+}
+
+// Reference holds additional information about CPE.
+type Reference struct {
+ URL string `xml:"href,attr"`
+ Desc string `xml:",chardata"`
+}
+
+// DeprecatedInfo contains the name that is deprecating the identifier name and the type of Deprecation
+type DeprecatedInfo struct {
+ Name NamePattern `xml:"name,attr"`
+ Type string `xml:"type,attr"`
+}
+
+// Deprecation contains the deprecation information for a specific deprecation of a given identifier name.
+type Deprecation struct {
+ Date time.Time `xml:"date,attr"`
+ DeprecatedBy []DeprecatedInfo `xml:"deprecated-by"`
+}
+
+// CPE23Item contains all CPE 2.3 specific data related to a given identifier name.
+type CPE23Item struct {
+ Name NamePattern `xml:"name,attr"`
+ Deprecation *Deprecation `xml:"deprecation"`
+ // TODO: implement ProvenanceRecord
+}
+
+// CPEItem contains all of the information for a single dictionary entry (identifier name), including metadata.
+type CPEItem struct {
+ Name NamePattern `xml:"name,attr"`
+ Deprecated bool `xml:"deprecated,attr"`
+ DeprecatedBy *NamePattern `xml:"deprecated_by,attr"`
+ DeprecationDate time.Time `xml:"deprecation_date,attr"`
+ CPE23 CPE23Item `xml:"cpe23-item"`
+ Title TextType `xml:"title"`
+ Notes TextType `xml:"notes"`
+ References []Reference `xml:"references>reference"`
+ // Calls out a check, such as an OVAL definition, that can confirm or reject
+ // an IT system as an instance of the named platform. 0-n occurrences.
+ // TODO: not implemented
+ Check struct{} `xml:"check"`
+}
+
+// Generator contains information about the generation of the dictionary file.
+type Generator struct {
+ ProductName string `xml:"product_name"`
+ ProductVersion string `xml:"product_version"`
+ SchemaVersion string `xml:"schema_version"`
+ TimeStamp time.Time `xml:"timestamp"`
+}
+
+// CPEList contains all of the dictionary entries and dictionary metadata.
+type CPEList struct {
+ Generator Generator `xml:"generator"`
+ Items []CPEItem `xml:"cpe-item"`
+}
+
+// Decode decodes dictionary XML
+func Decode(r io.Reader) (*CPEList, error) {
+ var list CPEList
+ if err := xml.NewDecoder(r).Decode(&list); err != nil {
+ return nil, err
+ }
+ return &list, nil
+}
diff --git a/server/vulnerabilities/nvd/tools/cpedict/cpedict_test.go b/server/vulnerabilities/nvd/tools/cpedict/cpedict_test.go
new file mode 100644
index 0000000000..3d5f6af345
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cpedict/cpedict_test.go
@@ -0,0 +1,101 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cpedict
+
+import (
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/wfn"
+)
+
+func TestDecode(t *testing.T) {
+ xmlStr := `
+
+
+
+ National Vulnerability Database (NVD)
+ 3.20
+ 2.3
+ 2018-04-25T03:50:11.922Z
+
+
+ $0.99 Kindle Books project $0.99 Kindle Books (aka com.kindle.books.for99) for android 6.0
+
+ Product information
+ Government Advisory
+
+
+
+
+ アドビシステムズ Flex
+ Adobe Flex
+
+
+
+ 3Com TippingPoint IPS TOS 2.1.3.6323
+
+
+
+
+
+
+
+`
+ data, err := Decode(strings.NewReader(xmlStr))
+ if err != nil {
+ t.Fatalf("failed to decode xml: %v", err)
+ }
+
+ gentm, _ := time.Parse(time.RFC3339, "2018-04-25T03:50:11.922Z")
+ generator := Generator{"National Vulnerability Database (NVD)", "3.20", "2.3", gentm}
+ if data.Generator != generator {
+ t.Errorf("bad generator:\n\texpected %+v\n\tgot %+v", data.Generator, generator)
+ }
+
+ wfname, _ := wfn.Parse("cpe:/a:%240.99_kindle_books_project:%240.99_kindle_books:6::~~~android~~")
+ item := data.Items[0]
+ if item.Name != item.CPE23.Name || item.Name != NamePattern(*wfname) {
+ t.Errorf("bad CPE name:\n\t2.2 is %+v\n\t2.3 is %+v", item.Name, item.CPE23.Name)
+ }
+ if len(item.References) != 2 {
+ t.Errorf("item was expected to have 2 references, %d found\n\t%v", len(item.References), item)
+ }
+
+ wfname, _ = wfn.Parse("cpe:2.3:o:3com:tippingpoint_ips_tos:2.1.3.6323:*:*:*:*:*:*:*")
+ name := NamePattern(*wfname)
+ deptm, _ := time.Parse(time.RFC3339, "2010-12-28T17:35:59.740Z")
+ item = data.Items[len(data.Items)-1]
+ if !item.Deprecated {
+ t.Errorf("item was expected to be deprecated, but isn't:\n\t%+v", item)
+ }
+ if !item.DeprecationDate.Equal(deptm) {
+ t.Errorf("item's deprecation time was expected to be\n\t%v\ngot\n\t%v", deptm, item.DeprecationDate)
+ }
+ if item.CPE23.Deprecation == nil {
+ t.Fatal("item was expected to have Deprecation info, but it doesn't")
+ }
+ dep := item.CPE23.Deprecation
+ if !dep.Date.Equal(item.DeprecationDate) {
+ t.Errorf("cpe23 deprecation date doesn't match the cpe22 one:\n\t%v\n\t%v", dep.Date, item.DeprecationDate)
+ }
+ if dep.DeprecatedBy[0].Name != name {
+ t.Errorf("item was expected to be deprecated by\n\t%v\n\tgot %v", dep.DeprecatedBy[0].Name, name)
+ }
+ if dep.DeprecatedBy[0].Type != "NAME_CORRECTION" {
+ t.Errorf("item was expected to be deprecated because of NAME_CORRECTION, got %v", dep.DeprecatedBy[0].Type)
+ }
+}
diff --git a/server/vulnerabilities/nvd/tools/cvefeed/cvecache.go b/server/vulnerabilities/nvd/tools/cvefeed/cvecache.go
new file mode 100644
index 0000000000..035bd94590
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cvefeed/cvecache.go
@@ -0,0 +1,287 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cvefeed
+
+import (
+ "sort"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "unsafe"
+
+ "github.com/facebookincubator/flog"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/wfn"
+)
+
+const cacheEvictPercentage = 0.1 // every eviction cycle invalidates this part of cache size at once
+
+// Index maps the CPEs to the entries in the NVD feed they mentioned in
+type Index map[string][]Vuln
+
+// NewIndex creates new Index from a slice of CVE entries
+func NewIndex(d Dictionary) Index {
+ idx := Index{}
+ for _, entry := range d {
+ set := map[string]bool{}
+ for _, cpe := range entry.Config() {
+ // Can happen, for instance, when the feed contains illegal binding of CPE name. Unfortunately, it happens to NVD,
+ // e.g. embedded ? in cpe:2.3:a:disney:where\\'s_my_perry?_free:1.5.1:*:*:*:*:android:*:* of CVE-2014-5606
+ if cpe == nil {
+ continue
+ }
+ product := cpe.Product
+ if wfn.HasWildcard(product) {
+ product = wfn.Any
+ }
+ if !set[product] {
+ set[product] = true
+ idx[product] = append(idx[product], entry)
+ }
+ }
+ }
+ return idx
+}
+
+// MatchResult stores CVE and a slice of CPEs that matched it
+type MatchResult struct {
+ CVE Vuln
+ CPEs []*wfn.Attributes
+}
+
+// cachedCVEs stores cached CVEs, a channel to signal if the value is ready
+type cachedCVEs struct {
+ res []MatchResult
+ ready chan struct{}
+ size int64
+ evictionIndex int // position in eviction queue
+}
+
+// updateResSize calculates the size of cached MatchResult and assigns it to cves.size
+func (cves *cachedCVEs) updateResSize(key string) {
+ if cves == nil {
+ return
+ }
+ cves.size = int64(int(unsafe.Sizeof(key)) + len(key))
+ cves.size += int64(unsafe.Sizeof(cves.res))
+ for i := range cves.res {
+ cves.size += int64(unsafe.Sizeof(cves.res[i].CVE))
+ for _, attr := range cves.res[i].CPEs {
+ cves.size += int64(len(attr.Part)) + int64(unsafe.Sizeof(attr.Part))
+ cves.size += int64(len(attr.Vendor)) + int64(unsafe.Sizeof(attr.Vendor))
+ cves.size += int64(len(attr.Product)) + int64(unsafe.Sizeof(attr.Product))
+ cves.size += int64(len(attr.Version)) + int64(unsafe.Sizeof(attr.Version))
+ cves.size += int64(len(attr.Update)) + int64(unsafe.Sizeof(attr.Update))
+ cves.size += int64(len(attr.Edition)) + int64(unsafe.Sizeof(attr.Edition))
+ cves.size += int64(len(attr.SWEdition)) + int64(unsafe.Sizeof(attr.SWEdition))
+ cves.size += int64(len(attr.TargetHW)) + int64(unsafe.Sizeof(attr.TargetHW))
+ cves.size += int64(len(attr.Other)) + int64(unsafe.Sizeof(attr.Other))
+ cves.size += int64(len(attr.Language)) + int64(unsafe.Sizeof(attr.Language))
+ }
+ }
+}
+
+// Cache caches CVEs for known CPEs
+type Cache struct {
+ // Used to compute the hit ratio
+ numLookups int64
+ numHits int64
+
+ // Actual cache data
+ data map[string]*cachedCVEs
+ evictionQ *evictionQueue
+ mu sync.Mutex
+ Dict Dictionary
+ Idx Index
+ MaxSize int64 // maximum size of the cache, 0 -- unlimited, -1 -- no caching
+ size int64 // current size of the cache
+ RequireVersion bool // ignore matching specifications that have Version == ANY
+}
+
+// NewCache creates new Cache instance with dictionary dict.
+func NewCache(dict Dictionary) *Cache {
+ return &Cache{Dict: dict, evictionQ: new(evictionQueue)}
+}
+
+// SetRequireVersion sets if the instance of cache fails matching the dictionary
+// records without Version attribute of CPE name.
+// Returns a pointer to the instance of Cache, for easy chaining.
+func (c *Cache) SetRequireVersion(requireVersion bool) *Cache {
+ c.RequireVersion = requireVersion
+ return c
+}
+
+// SetMaxSize sets maximum size of the cache to some pre-defined value,
+// size of 0 disables eviction (makes the cache grow indefinitely),
+// negative size disables caching.
+// Returns a pointer to the instance of Cache, for easy chaining.
+func (c *Cache) SetMaxSize(size int64) *Cache {
+ c.MaxSize = size
+ return c
+}
+
+// Get returns slice of CVEs for CPE names from cpes parameter;
+// if CVEs aren't cached (and the feature is enabled) it finds them in cveDict and caches the results
+func (c *Cache) Get(cpes []*wfn.Attributes) []MatchResult {
+ atomic.AddInt64(&c.numLookups, 1)
+
+ // negative max size of the cache disables caching
+ if c.MaxSize < 0 {
+ return c.match(cpes)
+ }
+
+ // otherwise, let's get to the business
+ key := cacheKey(cpes)
+ c.mu.Lock()
+ if c.data == nil {
+ c.data = make(map[string]*cachedCVEs)
+ }
+ cves := c.data[key]
+ if cves != nil {
+ atomic.AddInt64(&c.numHits, 1)
+
+ // value is being computed, wait till ready
+ c.mu.Unlock()
+ <-cves.ready
+ c.mu.Lock() // TODO: XXX: ugly, consider using atomic.Value instead
+ cves.evictionIndex = c.evictionQ.touch(cves.evictionIndex)
+ c.mu.Unlock()
+ return cves.res
+ }
+ // first request; the goroutine that sent it computes the value
+ cves = &cachedCVEs{ready: make(chan struct{})}
+ c.data[key] = cves
+ c.mu.Unlock()
+ // now other requests for same key wait on the channel, and the requests for the different keys aren't blocked
+ cves.res = c.match(cpes)
+ cves.updateResSize(key)
+ c.mu.Lock()
+ if c.MaxSize != 0 && c.size+cves.size > c.MaxSize {
+ c.evict(int64(cacheEvictPercentage*float64(c.MaxSize)) + cves.size)
+ }
+ c.size += cves.size
+ cves.evictionIndex = c.evictionQ.push(key)
+ c.mu.Unlock()
+ close(cves.ready)
+ return cves.res
+}
+
+// match will return all match results based on the given cpes
+func (c *Cache) match(cpes []*wfn.Attributes) []MatchResult {
+ d := c.Dict
+ if c.Idx != nil {
+ d = c.dictFromIndex(cpes)
+ }
+ return c.matchDict(cpes, d)
+}
+
+// dictFromIndex creates CVE dictionary from entries indexed by CPE names
+func (c *Cache) dictFromIndex(cpes []*wfn.Attributes) Dictionary {
+ d := Dictionary{}
+ if c.Idx == nil {
+ return d
+ }
+
+ knownEntries := map[Vuln]bool{}
+ addVulns := func(product string) {
+ for _, vuln := range c.Idx[product] {
+ if !knownEntries[vuln] {
+ knownEntries[vuln] = true
+ d[vuln.ID()] = vuln
+ }
+ }
+ }
+
+ for _, cpe := range cpes {
+ if cpe == nil { // should never happen
+ flog.Warning("nil CPE in list")
+ continue
+ }
+ // any of the CPEs having product=ANY would mean we need to match against the entire dictionary
+ if cpe.Product == wfn.Any {
+ return c.Dict
+ }
+ addVulns(cpe.Product)
+ }
+ addVulns(wfn.Any)
+
+ return d
+}
+
+// match matches the CPE names against internal vulnerability dictionary and returns a slice of matching resutls
+func (c *Cache) matchDict(cpes []*wfn.Attributes, dict Dictionary) (results []MatchResult) {
+ for _, v := range dict {
+ if matches := v.Match(cpes, c.RequireVersion); len(matches) > 0 {
+ results = append(results, MatchResult{v, matches})
+ }
+ }
+ return results
+}
+
+// evict the least recently used records untile nbytes of capacity is achieved or no more records left.
+// It is not concurrency-safe, c.mu should be locked before calling it.
+func (c *Cache) evict(nbytes int64) {
+ for c.size > 0 && c.size+nbytes > c.MaxSize {
+ key := c.evictionQ.pop()
+ cd, ok := c.data[key]
+ if !ok { // should not happen
+ panic("attempted to evict non-existent record")
+ }
+ c.size -= cd.size
+ delete(c.data, key)
+ }
+}
+
+func cacheKey(cpes []*wfn.Attributes) string {
+ parts := make([]string, 0, len(cpes))
+ for _, cpe := range cpes {
+ if cpe == nil {
+ continue
+ }
+ var out strings.Builder
+ out.WriteString(cpe.Part)
+ out.WriteByte('^')
+ out.WriteString(cpe.Vendor)
+ out.WriteByte('^')
+ out.WriteString(cpe.Product)
+ out.WriteByte('^')
+ out.WriteString(cpe.Version)
+ out.WriteByte('^')
+ out.WriteString(cpe.Update)
+ out.WriteByte('^')
+ out.WriteString(cpe.Edition)
+ out.WriteByte('^')
+ out.WriteString(cpe.SWEdition)
+ out.WriteByte('^')
+ out.WriteString(cpe.TargetSW)
+ out.WriteByte('^')
+ out.WriteString(cpe.TargetHW)
+ out.WriteByte('^')
+ out.WriteString(cpe.Other)
+ out.WriteByte('^')
+ out.WriteString(cpe.Language)
+ parts = append(parts, out.String())
+ }
+ sort.Strings(parts)
+ return strings.Join(parts, "#")
+}
+
+// HitRatio returns the cache hit ratio, the number of cache hits to the number
+// of lookups, as a percentage.
+func (c *Cache) HitRatio() float64 {
+ if c.numLookups == 0 {
+ return 0
+ }
+ return float64(c.numHits) / float64(c.numLookups) * 100
+}
diff --git a/server/vulnerabilities/nvd/tools/cvefeed/dictionary.go b/server/vulnerabilities/nvd/tools/cvefeed/dictionary.go
new file mode 100644
index 0000000000..3ddef4322a
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cvefeed/dictionary.go
@@ -0,0 +1,105 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cvefeed
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "strings"
+ "sync"
+)
+
+// Dictionary is a slice of entries
+type Dictionary map[string]Vuln
+
+// Override amends entries in Dictionary with configurations from Dictionary d2;
+// CVE will be matched if it matches the original config of d and does not match the config of d2.
+func (d *Dictionary) Override(d2 Dictionary) {
+ if d == nil {
+ return
+ }
+ if *d == nil {
+ *d = make(Dictionary)
+ }
+ for k, cve := range d2 {
+ if _, ok := (*d)[k]; ok {
+ (*d)[k] = OverrideVuln((*d)[k], cve)
+ }
+ }
+}
+
+// LoadJSONDictionary parses dictionary from multiple NVD vulnerability feed JSON files
+func LoadJSONDictionary(paths ...string) (Dictionary, error) {
+ return LoadFeed(loadJSONFile, paths...)
+}
+
+// LoadFeed calls loadFunc for each file in paths and returns the combined outputs in a Dictionary.
+func LoadFeed(loadFunc func(string) ([]Vuln, error), paths ...string) (Dictionary, error) {
+ dict := make(Dictionary)
+ var wg sync.WaitGroup
+ done := make(chan struct{})
+ errDone := make(chan struct{})
+ dictChan := make(chan []Vuln, 1)
+ errChan := make(chan error, 1)
+ for _, path := range paths {
+ wg.Add(1)
+ go func(path string) {
+ defer wg.Done()
+ feed, err := loadFunc(path)
+ if err != nil {
+ errChan <- fmt.Errorf("dictionary: failed to load feed %q: %v", path, err)
+ return
+ }
+ dictChan <- feed
+ }(path)
+ }
+ go func() {
+ for d := range dictChan {
+ for _, cve := range d {
+ if cveid := cve.ID(); cveid != "" {
+ dict[cveid] = cve
+ }
+ }
+ }
+ close(done)
+ }()
+ var errs []string
+ go func() {
+ for e := range errChan {
+ errs = append(errs, e.Error())
+ }
+ close(errDone)
+ }()
+ wg.Wait()
+ close(dictChan)
+ close(errChan)
+ <-done
+ <-errDone
+ if len(errs) > 0 {
+ return dict, errors.New(strings.Join(errs, "\n"))
+ }
+ return dict, nil
+}
+
+// loadJSONFile parses dictionary from NVD vulnerability feed JSON file
+func loadJSONFile(path string) ([]Vuln, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return nil, fmt.Errorf("dictionary: failed to load feed %q: %v", path, err)
+ }
+ defer f.Close()
+ return ParseJSON(f)
+}
diff --git a/server/vulnerabilities/nvd/tools/cvefeed/eviction_test.go b/server/vulnerabilities/nvd/tools/cvefeed/eviction_test.go
new file mode 100644
index 0000000000..595cc54645
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cvefeed/eviction_test.go
@@ -0,0 +1,123 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cvefeed
+
+import (
+ "bytes"
+ "sync"
+ "testing"
+
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/wfn"
+)
+
+func TestCacheEviction(t *testing.T) {
+ items, err := LoadFeed(func(_ string) ([]Vuln, error) {
+ return ParseJSON(bytes.NewBufferString(testJSONdict))
+ }, "")
+ if err != nil {
+ t.Fatalf("failed to parse the dictionary: %v", err)
+ }
+ cache := NewCache(items).SetMaxSize(2 * 1024)
+ matchingItem := &wfn.Attributes{Part: "a", Vendor: "microsoft", Product: "ie", Version: "5\\.4"}
+
+ // first, run concurrently and enjoy different sizes of cache logged on each run
+ var wg sync.WaitGroup
+ for i := 0; i < 50; i++ {
+ wg.Add(1)
+ go func(variant int) {
+ defer wg.Done()
+ inventory := []*wfn.Attributes{
+ matchingItem,
+ }
+ for i := 0; i < variant; i++ {
+ inventory = append(inventory, &wfn.Attributes{Vendor: "huh", Product: "brah"})
+ }
+ matches := cache.Get(inventory)
+ if len(matches) != 1 {
+ t.Errorf("variant %d: cache.Get() returned wrong amount of matches (%d, 1 was expected)", variant, len(matches))
+ return
+ }
+ if len(matches[0].CPEs) != 1 {
+ t.Errorf("variant %d: cache.Get() returned wrong a match with wrong number of CPEs (%d, 1 was expected)", variant, len(matches[0].CPEs))
+ }
+ if *matches[0].CPEs[0] != *matchingItem {
+ t.Errorf("variant %d: cache.Get() returned wrong match:\n%+v\n%+v was expected", variant, *matches[0].CPEs[0], *matchingItem)
+ }
+ }(i)
+ }
+ wg.Wait()
+ if cache.size > cache.MaxSize {
+ t.Errorf("concurrent run: cache size exceeds maximum: %d bytes out of %d bytes", cache.size, cache.MaxSize)
+ }
+ t.Logf("concurrent run: cache size %d/%d; %d records cached", cache.size, cache.MaxSize, len(cache.data))
+
+ // now let's get serious and get some deterministic resutls
+ for i := 0; i < 50; i++ {
+ variant := i
+ inventory := []*wfn.Attributes{
+ matchingItem,
+ }
+ for i := 0; i < variant; i++ {
+ inventory = append(inventory, &wfn.Attributes{Vendor: "huh", Product: "brah"})
+ }
+ matches := cache.Get(inventory)
+ if len(matches) != 1 {
+ t.Fatalf("variant %d: cache.Get() returned wrong amount of matches (%d, 1 was expected)", variant, len(matches))
+ }
+ if len(matches[0].CPEs) != 1 {
+ t.Errorf("variant %d: cache.Get() returned wrong a match with wrong number of CPEs (%d, 1 was expected)", variant, len(matches[0].CPEs))
+ }
+ if *matches[0].CPEs[0] != *matchingItem {
+ t.Errorf("variant %d: cache.Get() returned wrong match:\n%+v\n%+v was expected", variant, *matches[0].CPEs[0], *matchingItem)
+ }
+ }
+ if cache.size > cache.MaxSize {
+ t.Errorf("sequential run #1: cache size exceeds maximum: %d bytes out of %d bytes", cache.size, cache.MaxSize)
+ }
+ // the latest cached items are almost 1K long, so there should be only 1 left in the cache
+ if len(cache.data) > 1 {
+ t.Errorf("sequential run #1: more than 1 record cached (%d)", len(cache.data))
+ }
+ t.Logf("sequential run #1: cache size %d/%d; %d records cached", cache.size, cache.MaxSize, len(cache.data))
+
+ // and now let's go the other way around and make cache evict the bigger records first
+ for i := 39; i >= 0; i-- {
+ variant := i
+ inventory := []*wfn.Attributes{
+ matchingItem,
+ }
+ for i := 0; i < variant; i++ {
+ inventory = append(inventory, &wfn.Attributes{Vendor: "huh", Product: "brah"})
+ }
+ matches := cache.Get(inventory)
+ if len(matches) != 1 {
+ t.Errorf("variant %d: cache.Get() returned wrong amount of matches (%d, 1 was expected)", variant, len(matches))
+ }
+ if len(matches[0].CPEs) != 1 {
+ t.Errorf("variant %d: cache.Get() returned wrong a match with wrong number of CPEs (%d, 1 was expected)", variant, len(matches[0].CPEs))
+ }
+ if *matches[0].CPEs[0] != *matchingItem {
+ t.Errorf("variant %d: cache.Get() returned wrong match:\n%+v\n%+v was expected", variant, *matches[0].CPEs[0], *matchingItem)
+ }
+ }
+ if cache.size > cache.MaxSize {
+ t.Errorf("sequential run #2: cache size exceeds maximum: %d bytes out of %d bytes", cache.size, cache.MaxSize)
+ }
+ // Since we touch the smaller records first, we should have more of these cached
+ if len(cache.data) < 5 {
+ t.Errorf("sequential run #2: more than 1 record cached (%d)", len(cache.data))
+ }
+ t.Logf("sequential run #2: cache size %d/%d; %d records cached", cache.size, cache.MaxSize, len(cache.data))
+}
diff --git a/server/vulnerabilities/nvd/tools/cvefeed/evictionqueue.go b/server/vulnerabilities/nvd/tools/cvefeed/evictionqueue.go
new file mode 100644
index 0000000000..50267da109
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cvefeed/evictionqueue.go
@@ -0,0 +1,84 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cvefeed
+
+import (
+ "container/heap"
+ "time"
+)
+
+type evictionData struct {
+ key string // which key in Cache.data refers to it
+ index int // the index of item on the heap
+ access time.Time // last access time
+}
+
+// evictionQueue is a priority queue for LRU cache
+type evictionQueue struct {
+ q evictionHeap
+}
+
+// pop pops next key to evict
+func (eq *evictionQueue) pop() string {
+ if eq.q.Len() > 0 {
+ return heap.Pop(&eq.q).(*evictionData).key
+ }
+ return ""
+}
+
+// push pushes a key onto heap, returns the index item ended up at.
+func (eq *evictionQueue) push(key string) int {
+ index := eq.q.Len()
+ ed := &evictionData{
+ key: key,
+ index: index,
+ access: time.Now(),
+ }
+ heap.Push(&eq.q, ed)
+ return ed.index
+}
+
+// touch updates the access time of the item at index, returns the new index of that item.
+func (eq *evictionQueue) touch(index int) int {
+ ed := eq.q[index]
+ ed.access = time.Now()
+ heap.Fix(&eq.q, index)
+ return ed.index
+}
+
+// evictionHeap is a slice of evictionData that implements heap.Interface
+type evictionHeap []*evictionData
+
+func (eh evictionHeap) Len() int { return len(eh) }
+
+func (eh evictionHeap) Less(i, j int) bool { return eh[i].access.Before(eh[j].access) }
+
+func (eh evictionHeap) Swap(i, j int) {
+ eh[i], eh[j] = eh[j], eh[i]
+ eh[i].index, eh[j].index = i, j
+}
+
+func (eh *evictionHeap) Push(x interface{}) {
+ ed := x.(*evictionData)
+ ed.index = len(*eh)
+ *eh = append(*eh, ed)
+}
+
+func (eh *evictionHeap) Pop() interface{} {
+ old := *eh
+ ed := old[len(old)-1]
+ *eh = old[:len(old)-1]
+ return ed
+}
diff --git a/server/vulnerabilities/nvd/tools/cvefeed/evictionqueue_test.go b/server/vulnerabilities/nvd/tools/cvefeed/evictionqueue_test.go
new file mode 100644
index 0000000000..2ace2f299f
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cvefeed/evictionqueue_test.go
@@ -0,0 +1,84 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cvefeed
+
+import (
+ "testing"
+ "time"
+)
+
+func TestEvictionQueue(t *testing.T) {
+ var q evictionQueue
+ cases := []string{"hello", "world", "quux", "baz", "foo"}
+ for i, c := range cases {
+ idx := q.push(c)
+ if idx != i {
+ t.Errorf("push() returned wrong index %d (%d was expected)", idx, i)
+ }
+ time.Sleep(1 * time.Millisecond)
+ }
+
+ // first, it should appear in order
+ for i := range cases {
+ if cases[i] != q.q[i].key {
+ t.Errorf("unexpected queue order (before touch-ing):\nexpected %v\ngot %v", cases, listKeys(q.q))
+ break
+ }
+ }
+
+ // touch it in reverse order
+ for i := len(cases) - 1; i >= 0; i-- {
+ q.touch(i)
+ time.Sleep(1 * time.Millisecond)
+ }
+
+ // now baz and quux should be after foo and hello and world should be the last ones
+ // but the exact order is non-deterministic
+ for i, item := range q.q {
+ switch i {
+ case 0:
+ if item.key != "foo" {
+ t.Errorf("unexpected queue order (after touch-ing): %q at position %d", item.key, i)
+ }
+ case 1, 2:
+ if item.key != "baz" && item.key != "quux" {
+ t.Errorf("unexpected queue order (after touch-ing): %q at position %d", item.key, i)
+ }
+ case 3, 4:
+ if item.key != "hello" && item.key != "world" {
+ t.Errorf("unexpected queue order (after touch-ing): %q at position %d", item.key, i)
+ }
+ default:
+ t.Fatal("unreacheable code reached o_O")
+ }
+ }
+
+ // but when pop-ing the values from heap, it should come in order reverse to the one we started with
+ for i := len(cases) - 1; i >= 0; i-- {
+ item := q.pop()
+ if item != cases[i] {
+ t.Errorf("unexpected queue order (while pop-ing):\nexpected %v\ngot %v", cases, listKeys(q.q))
+ break
+ }
+ }
+}
+
+func listKeys(in []*evictionData) []string {
+ out := make([]string, len(in))
+ for i, item := range in {
+ out[i] = item.key
+ }
+ return out
+}
diff --git a/server/vulnerabilities/nvd/tools/cvefeed/feed.go b/server/vulnerabilities/nvd/tools/cvefeed/feed.go
new file mode 100644
index 0000000000..a4d7c23739
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cvefeed/feed.go
@@ -0,0 +1,85 @@
+// Package cvefeed defines types and methods necessary to parse NVD vulnerability
+// feed and match an inventory of CPE names against it.
+//
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package cvefeed provides an API to NVD CVE feeds parsing and matching.
+package cvefeed
+
+import (
+ "bufio"
+ "compress/bzip2"
+ "compress/gzip"
+ "encoding/json"
+ "fmt"
+ "io"
+ "io/ioutil"
+
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/cvefeed/nvd"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/cvefeed/nvd/schema"
+)
+
+// ParseJSON parses JSON dictionary from NVD vulnerability feed
+func ParseJSON(in io.Reader) ([]Vuln, error) {
+ feed, err := getFeed(in)
+ if err != nil {
+ return nil, fmt.Errorf("cvefeed.ParseJSON: %v", err)
+ }
+
+ vulns := make([]Vuln, 0, len(feed.CVEItems))
+ for _, cve := range feed.CVEItems {
+ if cve != nil && cve.Configurations != nil {
+ vulns = append(vulns, nvd.ToVuln(cve))
+ }
+ }
+ return vulns, nil
+}
+
+func getFeed(in io.Reader) (*schema.NVDCVEFeedJSON10, error) {
+ reader, err := setupReader(in)
+ if err != nil {
+ return nil, fmt.Errorf("can't setup reader: %v", err)
+ }
+ defer reader.Close()
+
+ var feed schema.NVDCVEFeedJSON10
+ if err := json.NewDecoder(reader).Decode(&feed); err != nil {
+ return nil, err
+ }
+ return &feed, nil
+}
+
+func setupReader(in io.Reader) (src io.ReadCloser, err error) {
+ r := bufio.NewReader(in)
+ header, err := r.Peek(2)
+ if err != nil {
+ return nil, err
+ }
+ // assume plain text first
+ src = ioutil.NopCloser(r)
+ // replace with gzip.Reader if gzip'ed
+ if header[0] == 0x1f && header[1] == 0x8b { // file is gzip'ed
+ zr, err := gzip.NewReader(r)
+ if err != nil {
+ return nil, err
+ }
+ src = zr
+ } else if header[0] == 'B' && header[1] == 'Z' {
+ // or with bzip2.Reader if bzip2'ed
+ src = ioutil.NopCloser(bzip2.NewReader(r))
+ }
+ // TODO: maybe support .zip
+ return src, nil
+}
diff --git a/server/vulnerabilities/nvd/tools/cvefeed/matching_json_test.go b/server/vulnerabilities/nvd/tools/cvefeed/matching_json_test.go
new file mode 100644
index 0000000000..baa73cad43
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cvefeed/matching_json_test.go
@@ -0,0 +1,300 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cvefeed
+
+import (
+ "bytes"
+ "fmt"
+ "testing"
+
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/wfn"
+)
+
+func TestBadJSONfeed(t *testing.T) {
+ items, err := ParseJSON(bytes.NewBufferString(testJSONdictBroken))
+ if err != nil {
+ t.Fatalf("failed to parse the dictionary: %v", err)
+ }
+ if len(items) > 0 {
+ t.Fatalf("expected the broken feed to be ignored, got %d items", len(items))
+ }
+}
+
+func TestMatchJSON(t *testing.T) {
+ cases := []struct {
+ Rule int
+ Inventory []*wfn.Attributes
+ Matches []*wfn.Attributes
+ }{
+ {
+ Rule: 0,
+ Inventory: []*wfn.Attributes{},
+ },
+ {
+ Rule: 0,
+ Inventory: []*wfn.Attributes{{}},
+ Matches: []*wfn.Attributes{{}},
+ },
+ {
+ Inventory: []*wfn.Attributes{
+ {Part: "o", Vendor: "linux", Product: "linux_kernel", Version: "2\\.6\\.1"},
+ {Part: "a", Vendor: "djvulibre_project", Product: "djvulibre", Version: "3\\.5\\.11"},
+ },
+ },
+ {
+ Rule: 0,
+ Inventory: []*wfn.Attributes{
+ {Part: "o", Vendor: "microsoft", Product: "windows_xp", Update: "sp3"},
+ {Part: "a", Vendor: "microsoft", Product: "ie", Version: "6\\.0"},
+ {Part: "a", Vendor: "facebook", Product: "styx", Version: "0\\.1"},
+ },
+ Matches: []*wfn.Attributes{
+ {Part: "o", Vendor: "microsoft", Product: "windows_xp", Update: "sp3"},
+ {Part: "a", Vendor: "microsoft", Product: "ie", Version: "6\\.0"},
+ },
+ },
+ {
+ Rule: 1,
+ Inventory: []*wfn.Attributes{{}},
+ Matches: []*wfn.Attributes{{}},
+ },
+ {
+ Rule: 1,
+ Inventory: []*wfn.Attributes{
+ {Part: "a", Vendor: "microsoft", Product: "ie", Version: "3\\.9"},
+ {Part: "a", Vendor: "microsoft", Product: "ie", Version: "4\\.0"},
+ {Part: "a", Vendor: "microsoft", Product: "ie", Version: "5\\.4"},
+ {Part: "a", Vendor: "microsoft", Product: "ie", Version: "6\\.0"},
+ },
+ Matches: []*wfn.Attributes{
+ {Part: "a", Vendor: "microsoft", Product: "ie", Version: "4\\.0"},
+ {Part: "a", Vendor: "microsoft", Product: "ie", Version: "5\\.4"},
+ },
+ },
+ {
+ Rule: 2,
+ Inventory: []*wfn.Attributes{{}},
+ Matches: []*wfn.Attributes{{}},
+ },
+ {
+ Rule: 2,
+ Inventory: []*wfn.Attributes{
+ {Part: "a", Vendor: "mozilla", Product: "firefox", Version: "64\\.0"},
+ },
+ },
+ }
+ items, err := ParseJSON(bytes.NewBufferString(testJSONdict))
+ if err != nil {
+ t.Fatalf("failed to parse the dictionary: %v", err)
+ }
+ for i, c := range cases {
+ t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+ mm := items[c.Rule].Match(c.Inventory, false)
+ if len(mm) != len(c.Matches) {
+ t.Fatalf("expected %d matches, got %d matches", len(mm), len(c.Matches))
+ }
+ if len(mm) > 0 && !matchesAll(mm, c.Matches) {
+ t.Fatalf("wrong match: expected %v, got %v", c.Matches, mm)
+ }
+ })
+ }
+}
+
+func TestMatchJSONrequireVersion(t *testing.T) {
+ inventory := []*wfn.Attributes{
+ {Part: "a", Vendor: "microsoft", Product: "ie", Version: "6\\.0"},
+ }
+ items, err := ParseJSON(bytes.NewBufferString(testJSONdict))
+ if err != nil {
+ t.Fatalf("failed to parse the dictionary: %v", err)
+ }
+ if mm := items[1].Match(inventory, true); len(mm) != 0 {
+ t.Fatal("platform was expected to be ignored because of absence of version, but matched")
+ }
+}
+
+func TestMatchJSONsmartVersionMatching(t *testing.T) {
+ inventory := []*wfn.Attributes{
+ {Part: "a", Vendor: "microsoft", Product: "ie", Version: "52\\.0"},
+ }
+ items, err := ParseJSON(bytes.NewBufferString(testJSONdict))
+ if err != nil {
+ t.Fatalf("failed to parse the dictionary: %v", err)
+ }
+ if mm := items[1].Match(inventory, true); len(mm) != 0 {
+ t.Errorf("version %q unexpectedly matched", inventory[0].Version)
+ }
+}
+
+func BenchmarkMatchJSON(b *testing.B) {
+ inventory := []*wfn.Attributes{
+ {Part: "o", Vendor: "microsoft", Product: "windows_xp", Update: "sp3"},
+ {Part: "a", Vendor: "microsoft", Product: "ie", Version: "6\\.0"},
+ {Part: "a", Vendor: "facebook", Product: "styx", Version: "0\\.1"},
+ }
+ items, err := ParseJSON(bytes.NewBufferString(testJSONdict))
+ if err != nil {
+ b.Fatalf("failed to parse the dictionary: %v", err)
+ }
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ if mm := items[0].Match(inventory, false); len(mm) == 0 {
+ b.Fatal("expected Match to match, it did not")
+ }
+ }
+}
+
+var testJSONdictBroken = `{
+ "CVE_data_format":"",
+ "CVE_data_type":"",
+ "CVE_data_version":"",
+ "CVE_Items":[
+ {},
+ {"cve":null},
+ {
+ "cve": {
+ "data_type" : "CVE",
+ "data_format" : "MITRE",
+ "data_version" : "4.0",
+ "CVE_data_meta" : {
+ "ID" : "TESTVE-2018-0001",
+ "ASSIGNER" : "cve@mitre.org"
+ }
+ },
+ "configurations": null
+ }
+ ]
+}
+`
+
+var testJSONdict = `{
+"CVE_data_type" : "CVE",
+"CVE_data_format" : "MITRE",
+"CVE_data_version" : "4.0",
+"CVE_data_numberOfCVEs" : "7083",
+"CVE_data_timestamp" : "2018-07-31T07:00Z",
+"CVE_Items" : [
+ {
+ "cve" : {
+ "data_type" : "CVE",
+ "data_format" : "MITRE",
+ "data_version" : "4.0",
+ "CVE_data_meta" : {
+ "ID" : "TESTVE-2018-0001",
+ "ASSIGNER" : "cve@mitre.org"
+ }
+ },
+ "configurations" : {
+ "CVE_data_version" : "4.0",
+ "nodes" : [
+ {
+ "operator" : "AND",
+ "children" : [
+ {
+ "operator" : "OR",
+ "cpe_match" : [ {
+ "vulnerable" : true,
+ "cpe22Uri" : "cpe:/a:microsoft:ie:6.%01",
+ "cpe23Uri" : "cpe:2.3:a:microsoft:ie:6.*:*:*:*:*:*:*:*"
+ } ]
+ },
+ {
+ "operator" : "OR",
+ "cpe_match" : [ {
+ "vulnerable" : true,
+ "cpe22Uri" : "cpe:/o:microsoft:windows_xp::sp%02",
+ "cpe23Uri" : "cpe:2.3:o:microsoft:windows_xp:*:sp?:*:*:*:*:*:*"
+ } ]
+ }
+ ]
+ }
+ ]
+ }
+ },
+ {
+ "cve" : {
+ "data_type" : "CVE",
+ "data_format" : "MITRE",
+ "data_version" : "4.0",
+ "CVE_data_meta" : {
+ "ID" : "TESTVE-2018-0002",
+ "ASSIGNER" : "cve@mitre.org"
+ }
+ },
+ "configurations" : {
+ "CVE_data_version" : "4.0",
+ "nodes" : [
+ {
+ "operator" : "AND",
+ "children" : [
+ {
+ "operator" : "OR",
+ "cpe_match" : [ {
+ "vulnerable" : true,
+ "cpe22Uri" : "cpe:/a:microsoft:ie",
+ "cpe23Uri" : "cpe:2.3:a:microsoft:ie:*:*:*:*:*:*:*:*",
+ "versionStartIncluding" : "4.0",
+ "versionEndExcluding" : "6.0"
+ } ]
+ }
+ ]
+ }
+ ]
+ }
+ },
+ {
+ "cve": {
+ "data_format": "MITRE",
+ "data_type": "CVE",
+ "data_version": "4.0",
+ "CVE_data_meta": {
+ "ASSIGNER": "cve@mitre.org",
+ "ID": "CVE-2002-2436"
+ }
+ },
+ "configurations": {
+ "CVE_data_version": "4.0",
+ "nodes": [
+ {
+ "cpe_match": [
+ {
+ "cpe23Uri": "cpe:2.3:a:mozilla:firefox:*:*:*:*:*:*:*:*",
+ "versionEndIncluding": "3.6.24",
+ "vulnerable": true
+ }
+ ],
+ "operator": "OR"
+ }
+ ]
+ }
+ }
+] }`
+
+func matchesAll(src, tgt []*wfn.Attributes) bool {
+ if len(src) != len(tgt) {
+ return false
+ }
+ for i, j := 0, 0; i < len(src); i, j = i+1, 0 {
+ for ; j < len(tgt); j++ {
+ if *src[i] == *tgt[j] {
+ break
+ }
+ }
+ if j == len(tgt) { // reached the end, no match
+ return false
+ }
+ }
+ return true
+}
diff --git a/server/vulnerabilities/nvd/tools/cvefeed/matching_overrides_test.go b/server/vulnerabilities/nvd/tools/cvefeed/matching_overrides_test.go
new file mode 100644
index 0000000000..1dc542bd23
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cvefeed/matching_overrides_test.go
@@ -0,0 +1,111 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cvefeed
+
+import (
+ "bytes"
+ "fmt"
+ "testing"
+
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/wfn"
+)
+
+func TestMatchOverrides(t *testing.T) {
+ cases := [][]*wfn.Attributes{
+ {},
+ {
+ {Part: "o", Vendor: "linux", Product: "linux_kernel", Version: "2\\.6\\.1"},
+ {Part: "a", Vendor: "djvulibre_project", Product: "djvulibre", Version: "3\\.5\\.11"},
+ },
+ {
+ {Part: "o", Vendor: "microsoft", Product: "windows_xp", Update: "sp3"},
+ {Part: "a", Vendor: "microsoft", Product: "ie", Version: "5\\.0", Update: wfn.NA},
+ },
+ {
+ {Part: "o", Vendor: "microsoft", Product: "windows_xp", Update: "sp3"},
+ {Part: "a", Vendor: "microsoft", Product: "ie", Version: "5\\.0", Update: "patched"},
+ },
+ }
+ dict, err := LoadFeed(func(_ string) ([]Vuln, error) {
+ return ParseJSON(bytes.NewBufferString(testJSONdict))
+ }, "")
+ if err != nil {
+ t.Fatalf("could not load test JSON feed: %v", err)
+ }
+ original, _ := LoadFeed(func(_ string) ([]Vuln, error) {
+ return ParseJSON(bytes.NewBufferString(testJSONdict))
+ }, "")
+ overrides, err := LoadFeed(func(_ string) ([]Vuln, error) {
+ return ParseJSON(bytes.NewBufferString(testJSONoverride))
+ }, "")
+ if err != nil {
+ t.Fatalf("could not load test overrides: %v", err)
+ }
+ dict.Override(overrides)
+
+ for n, c := range cases {
+ c := c
+ t.Run(fmt.Sprintf("%d", n), func(t *testing.T) {
+ var matchOriginal, matchOverride, matchDict bool
+ if m := dict["TESTVE-2018-0002"].Match(c, false); len(m) > 0 {
+ matchDict = true
+ }
+ if m := original["TESTVE-2018-0002"].Match(c, false); len(m) > 0 {
+ matchOriginal = true
+ }
+ if m := overrides["TESTVE-2018-0002"].Match(c, false); len(m) > 0 {
+ matchOverride = true
+ }
+ if matchOriginal && matchDict && matchOverride {
+ t.Fatal("case was not overriden")
+ } else if matchDict && !matchOriginal {
+ t.Fatal("unexpected match")
+ }
+ })
+ }
+}
+
+var testJSONoverride = `{
+"CVE_data_type" : "CVE",
+"CVE_data_format" : "MITRE",
+"CVE_data_version" : "4.0",
+"CVE_data_numberOfCVEs" : "7083",
+"CVE_data_timestamp" : "2018-07-31T07:00Z",
+"CVE_Items" : [
+ {
+ "cve" : {
+ "data_type" : "CVE",
+ "data_format" : "MITRE",
+ "data_version" : "4.0",
+ "CVE_data_meta" : {
+ "ID" : "TESTVE-2018-0002",
+ "ASSIGNER" : "cve@mitre.org"
+ }
+ },
+ "configurations" : {
+ "CVE_data_version" : "4.0",
+ "nodes" : [
+ {
+ "operator" : "OR",
+ "cpe_match" : [ {
+ "vulnerable" : true,
+ "cpe23Uri" : "cpe:2.3:a:microsoft:ie:*:patched:*:*:*:*:*:*"
+ } ]
+ }
+ ]
+ }
+ }
+]
+}`
diff --git a/server/vulnerabilities/nvd/tools/cvefeed/nvd/match_cpe.go b/server/vulnerabilities/nvd/tools/cvefeed/nvd/match_cpe.go
new file mode 100644
index 0000000000..636a78a93a
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cvefeed/nvd/match_cpe.go
@@ -0,0 +1,141 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nvd
+
+import (
+ "fmt"
+
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/cvefeed/nvd/schema"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/wfn"
+)
+
+// cpeMatch is a wrapper around the actual NVDCVEFeedJSON10DefCPEMatch
+type cpeMatch struct {
+ *wfn.Attributes
+ vulnerable bool
+ versionEndExcluding string
+ versionEndIncluding string
+ versionStartExcluding string
+ versionStartIncluding string
+ hasVersionRanges bool
+}
+
+// Matcher returns an object which knows how to match attributes
+func cpeMatcher(ID string, nvdMatch *schema.NVDCVEFeedJSON10DefCPEMatch) (wfn.Matcher, error) {
+ parse := func(uri string) (*wfn.Attributes, error) {
+ if uri == "" {
+ return nil, fmt.Errorf("%s: can't parse empty uri", ID)
+ }
+ return wfn.Parse(uri)
+ }
+
+ // parse
+ match := cpeMatch{vulnerable: nvdMatch.Vulnerable}
+ var err error
+ if match.Attributes, err = parse(nvdMatch.Cpe23Uri); err != nil {
+ if match.Attributes, err = parse(nvdMatch.Cpe22Uri); err != nil {
+ return nil, fmt.Errorf("%s: unable to parse both cpe2.2 and cpe2.3", ID)
+ }
+ }
+
+ match.versionEndExcluding = nvdMatch.VersionEndExcluding
+ match.versionEndIncluding = nvdMatch.VersionEndIncluding
+ match.versionStartExcluding = nvdMatch.VersionStartExcluding
+ match.versionStartIncluding = nvdMatch.VersionStartIncluding
+
+ if match.versionStartIncluding != "" || match.versionStartExcluding != "" ||
+ match.versionEndIncluding != "" || match.versionEndExcluding != "" {
+ match.hasVersionRanges = true
+ }
+
+ return &match, nil
+}
+
+// Match is part of the Matcher interface
+func (cm *cpeMatch) Match(attrs []*wfn.Attributes, requireVersion bool) (matches []*wfn.Attributes) {
+ for _, attr := range attrs {
+ if cm.match(attr, requireVersion) {
+ matches = append(matches, attr)
+ }
+ }
+ return matches
+}
+
+// Match implements wfn.Matcher interface
+func (cm *cpeMatch) match(attr *wfn.Attributes, requireVersion bool) bool {
+ if cm == nil || cm.Attributes == nil {
+ return false
+ }
+
+ if requireVersion {
+ // if we require version, then we need either version ranges or version not to be *
+ if !cm.hasVersionRanges && cm.Attributes.Version == wfn.Any {
+ return false
+ }
+ }
+
+ // here we have a version: either actual one or ranges
+
+ // check whether everything except for version matches
+ if !cm.Attributes.MatchWithoutVersion(attr) {
+ return false
+ }
+
+ if cm.Attributes.Version == wfn.Any {
+ if !cm.hasVersionRanges {
+ // if version is any and doesn't have version ranges, then it matches any
+ return !requireVersion
+ } // otherwise we try to match it at the end of the function
+ } else if cm.Attributes.MatchOnlyVersion(attr) {
+ return true // version matched
+ }
+
+ // if it got to here, it means:
+ // - matched attr without version
+ // - didn't match version, or require version was set and version was *
+
+ if attr.Version == wfn.Any {
+ return true
+ }
+
+ if !cm.hasVersionRanges {
+ return false
+ }
+
+ // if hasVersionRanges and attr version is NA, then return false
+ if attr.Version == wfn.NA {
+ return false
+ }
+
+ // match version to ranges
+ ver := wfn.StripSlashes(attr.Version)
+
+ matches := true
+
+ if cm.versionStartIncluding != "" {
+ matches = matches && smartVerCmp(ver, cm.versionStartIncluding) >= 0
+ }
+ if cm.versionStartExcluding != "" {
+ matches = matches && smartVerCmp(ver, cm.versionStartExcluding) > 0
+ }
+ if cm.versionEndIncluding != "" {
+ matches = matches && smartVerCmp(ver, cm.versionEndIncluding) <= 0
+ }
+ if cm.versionEndExcluding != "" {
+ matches = matches && smartVerCmp(ver, cm.versionEndExcluding) < 0
+ }
+
+ return matches
+}
diff --git a/server/vulnerabilities/nvd/tools/cvefeed/nvd/match_cve.go b/server/vulnerabilities/nvd/tools/cvefeed/nvd/match_cve.go
new file mode 100644
index 0000000000..f6b789f837
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cvefeed/nvd/match_cve.go
@@ -0,0 +1,176 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nvd
+
+import (
+ "regexp"
+
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/cvefeed/nvd/schema"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/wfn"
+)
+
+var cveRegex = regexp.MustCompile("CVE-[0-9]{4}-[0-9]{4,}")
+
+func ToVuln(cve *schema.NVDCVEFeedJSON10DefCVEItem) *Vuln {
+ vuln := &Vuln{
+ cveItem: cve,
+ }
+
+ var ms []wfn.Matcher
+ for _, node := range cve.Configurations.Nodes {
+ if node != nil {
+ if m, err := nodeMatcher(vuln.ID(), node); err == nil {
+ ms = append(ms, m)
+ }
+ }
+ }
+ vuln.Matcher = wfn.MatchAny(ms...)
+
+ return vuln
+}
+
+// Vuln implements the cvefeed.Vuln interface
+type Vuln struct {
+ cveItem *schema.NVDCVEFeedJSON10DefCVEItem
+ wfn.Matcher
+}
+
+// Schema returns the underlying schema of the Vuln
+func (v *Vuln) Schema() *schema.NVDCVEFeedJSON10DefCVEItem {
+ return v.cveItem
+}
+
+// ID is a part of the cvefeed.Vuln Interface
+func (v *Vuln) ID() string {
+ if v == nil || v.cveItem == nil || v.cveItem.CVE == nil || v.cveItem.CVE.CVEDataMeta == nil {
+ return ""
+ }
+ return v.cveItem.CVE.CVEDataMeta.ID
+}
+
+// CVEs is a part of the cvefeed.Vuln Interface
+func (v *Vuln) CVEs() []string {
+ if v == nil || v.cveItem == nil || v.cveItem.CVE == nil {
+ return nil
+ }
+
+ var cves []string
+
+ addMatch := func(s string) bool {
+ if cve := cveRegex.FindString(s); cve != "" {
+ cves = append(cves, cve)
+ return true
+ }
+ return false
+ }
+
+ // check if ID contains CVE
+ addMatch(v.ID())
+
+ // add references
+ if refs := v.cveItem.CVE.References; refs != nil {
+ for _, refd := range refs.ReferenceData {
+ if refd != nil {
+ addMatch(refd.Name)
+ }
+ }
+ }
+
+ return unique(cves)
+}
+
+// CWEs is a part of the cvefeed.Vuln Interface
+func (v *Vuln) CWEs() []string {
+ if v == nil || v.cveItem == nil || v.cveItem.CVE == nil || v.cveItem.CVE.Problemtype == nil {
+ return nil
+ }
+
+ var cwes []string
+
+ for _, ptd := range v.cveItem.CVE.Problemtype.ProblemtypeData {
+ if ptd != nil {
+ for _, desc := range ptd.Description {
+ if desc != nil {
+ if desc.Lang == "en" {
+ cwes = append(cwes, desc.Value)
+ }
+ }
+ }
+ }
+ }
+
+ return unique(cwes)
+}
+
+// CVSSv2BaseScore is a part of the cvefeed.Vuln Interface
+func (v *Vuln) CVSSv2BaseScore() float64 {
+ if c := v.cvssv2(); c != nil {
+ return c.BaseScore
+ }
+ return 0.0
+}
+
+// CVSSv2Vector is a part of the cvefeed.Vuln Interface
+func (v *Vuln) CVSSv2Vector() string {
+ if c := v.cvssv2(); c != nil {
+ return c.VectorString
+ }
+ return ""
+}
+
+// CVSSv3BaseScore is a part of the cvefeed.Vuln Interface
+func (v *Vuln) CVSSv3BaseScore() float64 {
+ if c := v.cvssv3(); c != nil {
+ return c.BaseScore
+ }
+ return 0.0
+}
+
+// CVSSv3Vector is a part of the cvefeed.Vuln Interface
+func (v *Vuln) CVSSv3Vector() string {
+ if c := v.cvssv3(); c != nil {
+ return c.VectorString
+ }
+ return ""
+}
+
+// unique returns unique strings from input
+func unique(ss []string) []string {
+ var us []string
+ set := make(map[string]bool)
+ for _, s := range ss {
+ if !set[s] {
+ us = append(us, s)
+ }
+ set[s] = true
+ }
+ return us
+}
+
+// just a helper to return the cvssv2 data
+func (v *Vuln) cvssv2() *schema.CVSSV20 {
+ if v == nil || v.cveItem == nil || v.cveItem.Impact == nil || v.cveItem.Impact.BaseMetricV2 == nil {
+ return nil
+ }
+ return v.cveItem.Impact.BaseMetricV2.CVSSV2
+}
+
+// just a helper to return the cvssv3 data
+func (v *Vuln) cvssv3() *schema.CVSSV30 {
+ if v == nil || v.cveItem == nil || v.cveItem.Impact == nil || v.cveItem.Impact.BaseMetricV3 == nil {
+ return nil
+ }
+ return v.cveItem.Impact.BaseMetricV3.CVSSV3
+}
diff --git a/server/vulnerabilities/nvd/tools/cvefeed/nvd/match_node.go b/server/vulnerabilities/nvd/tools/cvefeed/nvd/match_node.go
new file mode 100644
index 0000000000..e1a1b988bf
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cvefeed/nvd/match_node.go
@@ -0,0 +1,69 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nvd
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/facebookincubator/flog"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/cvefeed/nvd/schema"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/wfn"
+)
+
+// Matcher returns an object which knows how to match attributes
+func nodeMatcher(ID string, node *schema.NVDCVEFeedJSON10DefNode) (wfn.Matcher, error) {
+ if node == nil {
+ return nil, fmt.Errorf("%s: node is nil", ID)
+ }
+
+ var ms []wfn.Matcher
+ for _, match := range node.CPEMatch {
+ if match != nil {
+ if m, err := cpeMatcher(ID, match); err == nil {
+ ms = append(ms, m)
+ }
+ }
+ }
+ for _, child := range node.Children {
+ if child != nil {
+ if m, err := nodeMatcher(ID, child); err == nil {
+ ms = append(ms, m)
+ }
+ }
+ }
+
+ if len(ms) == 0 {
+ return nil, fmt.Errorf("%s: empty configuration for node", ID)
+ }
+
+ var m wfn.Matcher
+
+ switch strings.ToUpper(node.Operator) {
+ default:
+ flog.Warningf("%s: unknown operator, defaulting to OR: got %q", ID, node.Operator)
+ fallthrough
+ case "OR":
+ m = wfn.MatchAny(ms...)
+ case "AND":
+ m = wfn.MatchAll(ms...)
+ }
+
+ if node.Negate {
+ m = wfn.DontMatch(m)
+ }
+
+ return m, nil
+}
diff --git a/server/vulnerabilities/nvd/tools/cvefeed/nvd/schema/schema.go b/server/vulnerabilities/nvd/tools/cvefeed/nvd/schema/schema.go
new file mode 100644
index 0000000000..ba17bffaba
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cvefeed/nvd/schema/schema.go
@@ -0,0 +1,255 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schema
+
+// TimeLayout is the layout of NVD CVE timestamps.
+const TimeLayout = "2006-01-02T15:04Z"
+
+// NVDCVEFeedJSON10DefCPEName was auto-generated.
+// CPE name.
+type NVDCVEFeedJSON10DefCPEName struct {
+ Cpe22Uri string `json:"cpe22Uri,omitempty"`
+ Cpe23Uri string `json:"cpe23Uri"`
+}
+
+// NVDCVEFeedJSON10DefCPEMatch was auto-generated.
+// CPE match string or range.
+type NVDCVEFeedJSON10DefCPEMatch struct {
+ CPEName []*NVDCVEFeedJSON10DefCPEName `json:"cpe_name,omitempty"`
+ Cpe22Uri string `json:"cpe22Uri,omitempty"`
+ Cpe23Uri string `json:"cpe23Uri"`
+ VersionEndExcluding string `json:"versionEndExcluding,omitempty"`
+ VersionEndIncluding string `json:"versionEndIncluding,omitempty"`
+ VersionStartExcluding string `json:"versionStartExcluding,omitempty"`
+ VersionStartIncluding string `json:"versionStartIncluding,omitempty"`
+ Vulnerable bool `json:"vulnerable"`
+}
+
+// NVDCVEFeedJSON10DefNode was auto-generated.
+// Defines a node or sub-node in an NVD applicability statement.
+type NVDCVEFeedJSON10DefNode struct {
+ CPEMatch []*NVDCVEFeedJSON10DefCPEMatch `json:"cpe_match,omitempty"`
+ Children []*NVDCVEFeedJSON10DefNode `json:"children,omitempty"`
+ Negate bool `json:"negate,omitempty"`
+ Operator string `json:"operator,omitempty"`
+}
+
+// NVDCVEFeedJSON10DefConfigurations was auto-generated.
+// Defines the set of product configurations for a NVD applicability statement.
+type NVDCVEFeedJSON10DefConfigurations struct {
+ CVEDataVersion string `json:"CVE_data_version"`
+ Nodes []*NVDCVEFeedJSON10DefNode `json:"nodes,omitempty"`
+}
+
+// CVEJSON40CVEDataMeta was auto-generated.
+type CVEJSON40CVEDataMeta struct {
+ ASSIGNER string `json:"ASSIGNER"`
+ ID string `json:"ID"`
+ STATE string `json:"STATE,omitempty"`
+}
+
+// CVEJSON40ProductVersionVersionData was auto-generated.
+type CVEJSON40ProductVersionVersionData struct {
+ VersionAffected string `json:"version_affected,omitempty"`
+ VersionValue string `json:"version_value"`
+}
+
+// CVEJSON40ProductVersion was auto-generated.
+type CVEJSON40ProductVersion struct {
+ VersionData []*CVEJSON40ProductVersionVersionData `json:"version_data"`
+}
+
+// CVEJSON40Product was auto-generated.
+type CVEJSON40Product struct {
+ ProductName string `json:"product_name"`
+ Version *CVEJSON40ProductVersion `json:"version"`
+}
+
+// CVEJSON40AffectsVendorVendorDataProduct was auto-generated.
+type CVEJSON40AffectsVendorVendorDataProduct struct {
+ ProductData []*CVEJSON40Product `json:"product_data"`
+}
+
+// CVEJSON40AffectsVendorVendorData was auto-generated.
+type CVEJSON40AffectsVendorVendorData struct {
+ Product *CVEJSON40AffectsVendorVendorDataProduct `json:"product"`
+ VendorName string `json:"vendor_name"`
+}
+
+// CVEJSON40AffectsVendor was auto-generated.
+type CVEJSON40AffectsVendor struct {
+ VendorData []*CVEJSON40AffectsVendorVendorData `json:"vendor_data"`
+}
+
+// CVEJSON40Affects was auto-generated.
+type CVEJSON40Affects struct {
+ Vendor *CVEJSON40AffectsVendor `json:"vendor"`
+}
+
+// CVEJSON40LangString was auto-generated.
+type CVEJSON40LangString struct {
+ Lang string `json:"lang"`
+ Value string `json:"value"`
+}
+
+// CVEJSON40Description was auto-generated.
+type CVEJSON40Description struct {
+ DescriptionData []*CVEJSON40LangString `json:"description_data"`
+}
+
+// CVEJSON40ProblemtypeProblemtypeData was auto-generated.
+type CVEJSON40ProblemtypeProblemtypeData struct {
+ Description []*CVEJSON40LangString `json:"description"`
+}
+
+// CVEJSON40Problemtype was auto-generated.
+type CVEJSON40Problemtype struct {
+ ProblemtypeData []*CVEJSON40ProblemtypeProblemtypeData `json:"problemtype_data"`
+}
+
+// CVEJSON40Reference was auto-generated.
+type CVEJSON40Reference struct {
+ Name string `json:"name,omitempty"`
+ Refsource string `json:"refsource,omitempty"`
+ Tags []string `json:"tags,omitempty"`
+ URL string `json:"url"`
+}
+
+// CVEJSON40References was auto-generated.
+type CVEJSON40References struct {
+ ReferenceData []*CVEJSON40Reference `json:"reference_data"`
+}
+
+// CVEJSON40 was auto-generated.
+// Source: https://csrc.nist.gov/schema/nvd/feed/1.0/CVE_JSON_4.0_min.schema
+type CVEJSON40 struct {
+ Affects *CVEJSON40Affects `json:"affects"`
+ CVEDataMeta *CVEJSON40CVEDataMeta `json:"CVE_data_meta"`
+ DataFormat string `json:"data_format"`
+ DataType string `json:"data_type"`
+ DataVersion string `json:"data_version"`
+ Description *CVEJSON40Description `json:"description"`
+ Problemtype *CVEJSON40Problemtype `json:"problemtype"`
+ References *CVEJSON40References `json:"references"`
+}
+
+// CVSSV20 was auto-generated.
+// Source: https://csrc.nist.gov/schema/nvd/feed/1.0/cvss-v2.0.json
+type CVSSV20 struct {
+ AccessComplexity string `json:"accessComplexity,omitempty"`
+ AccessVector string `json:"accessVector,omitempty"`
+ Authentication string `json:"authentication,omitempty"`
+ AvailabilityImpact string `json:"availabilityImpact,omitempty"`
+ AvailabilityRequirement string `json:"availabilityRequirement,omitempty"`
+ BaseScore float64 `json:"baseScore"`
+ CollateralDamagePotential string `json:"collateralDamagePotential,omitempty"`
+ ConfidentialityImpact string `json:"confidentialityImpact,omitempty"`
+ ConfidentialityRequirement string `json:"confidentialityRequirement,omitempty"`
+ EnvironmentalScore float64 `json:"environmentalScore,omitempty"`
+ Exploitability string `json:"exploitability,omitempty"`
+ IntegrityImpact string `json:"integrityImpact,omitempty"`
+ IntegrityRequirement string `json:"integrityRequirement,omitempty"`
+ RemediationLevel string `json:"remediationLevel,omitempty"`
+ ReportConfidence string `json:"reportConfidence,omitempty"`
+ TargetDistribution string `json:"targetDistribution,omitempty"`
+ TemporalScore float64 `json:"temporalScore,omitempty"`
+ VectorString string `json:"vectorString"`
+ Version string `json:"version"`
+}
+
+// NVDCVEFeedJSON10DefImpactBaseMetricV2 was auto-generated.
+// CVSS V2.0 score.
+type NVDCVEFeedJSON10DefImpactBaseMetricV2 struct {
+ AcInsufInfo bool `json:"acInsufInfo,omitempty"`
+ CVSSV2 *CVSSV20 `json:"cvssV2,omitempty"`
+ ExploitabilityScore float64 `json:"exploitabilityScore,omitempty"`
+ ImpactScore float64 `json:"impactScore,omitempty"`
+ ObtainAllPrivilege bool `json:"obtainAllPrivilege,omitempty"`
+ ObtainOtherPrivilege bool `json:"obtainOtherPrivilege,omitempty"`
+ ObtainUserPrivilege bool `json:"obtainUserPrivilege,omitempty"`
+ Severity string `json:"severity,omitempty"`
+ UserInteractionRequired bool `json:"userInteractionRequired,omitempty"`
+}
+
+// CVSSV30 was auto-generated.
+// Source: https://csrc.nist.gov/schema/nvd/feed/1.0/cvss-v3.0.json
+type CVSSV30 struct {
+ AttackComplexity string `json:"attackComplexity,omitempty"`
+ AttackVector string `json:"attackVector,omitempty"`
+ AvailabilityImpact string `json:"availabilityImpact,omitempty"`
+ AvailabilityRequirement string `json:"availabilityRequirement,omitempty"`
+ BaseScore float64 `json:"baseScore"`
+ BaseSeverity string `json:"baseSeverity"`
+ ConfidentialityImpact string `json:"confidentialityImpact,omitempty"`
+ ConfidentialityRequirement string `json:"confidentialityRequirement,omitempty"`
+ EnvironmentalScore float64 `json:"environmentalScore,omitempty"`
+ EnvironmentalSeverity string `json:"environmentalSeverity,omitempty"`
+ ExploitCodeMaturity string `json:"exploitCodeMaturity,omitempty"`
+ IntegrityImpact string `json:"integrityImpact,omitempty"`
+ IntegrityRequirement string `json:"integrityRequirement,omitempty"`
+ ModifiedAttackComplexity string `json:"modifiedAttackComplexity,omitempty"`
+ ModifiedAttackVector string `json:"modifiedAttackVector,omitempty"`
+ ModifiedAvailabilityImpact string `json:"modifiedAvailabilityImpact,omitempty"`
+ ModifiedConfidentialityImpact string `json:"modifiedConfidentialityImpact,omitempty"`
+ ModifiedIntegrityImpact string `json:"modifiedIntegrityImpact,omitempty"`
+ ModifiedPrivilegesRequired string `json:"modifiedPrivilegesRequired,omitempty"`
+ ModifiedScope string `json:"modifiedScope,omitempty"`
+ ModifiedUserInteraction string `json:"modifiedUserInteraction,omitempty"`
+ PrivilegesRequired string `json:"privilegesRequired,omitempty"`
+ RemediationLevel string `json:"remediationLevel,omitempty"`
+ ReportConfidence string `json:"reportConfidence,omitempty"`
+ Scope string `json:"scope,omitempty"`
+ TemporalScore float64 `json:"temporalScore,omitempty"`
+ TemporalSeverity string `json:"temporalSeverity,omitempty"`
+ UserInteraction string `json:"userInteraction,omitempty"`
+ VectorString string `json:"vectorString"`
+ Version string `json:"version"`
+}
+
+// NVDCVEFeedJSON10DefImpactBaseMetricV3 was auto-generated.
+// CVSS V3.0 score.
+type NVDCVEFeedJSON10DefImpactBaseMetricV3 struct {
+ CVSSV3 *CVSSV30 `json:"cvssV3,omitempty"`
+ ExploitabilityScore float64 `json:"exploitabilityScore,omitempty"`
+ ImpactScore float64 `json:"impactScore,omitempty"`
+}
+
+// NVDCVEFeedJSON10DefImpact was auto-generated.
+// Impact scores for a vulnerability as found on NVD.
+type NVDCVEFeedJSON10DefImpact struct {
+ BaseMetricV2 *NVDCVEFeedJSON10DefImpactBaseMetricV2 `json:"baseMetricV2,omitempty"`
+ BaseMetricV3 *NVDCVEFeedJSON10DefImpactBaseMetricV3 `json:"baseMetricV3,omitempty"`
+}
+
+// NVDCVEFeedJSON10DefCVEItem was auto-generated.
+// Defines a vulnerability in the NVD data feed.
+type NVDCVEFeedJSON10DefCVEItem struct {
+ CVE *CVEJSON40 `json:"cve"`
+ Configurations *NVDCVEFeedJSON10DefConfigurations `json:"configurations,omitempty"`
+ Impact *NVDCVEFeedJSON10DefImpact `json:"impact,omitempty"`
+ LastModifiedDate string `json:"lastModifiedDate,omitempty"`
+ PublishedDate string `json:"publishedDate,omitempty"`
+}
+
+// NVDCVEFeedJSON10 was auto-generated.
+// Source: https://csrc.nist.gov/schema/nvd/feed/1.0/nvd_cve_feed_json_1.0.schema
+type NVDCVEFeedJSON10 struct {
+ CVEDataFormat string `json:"CVE_data_format"`
+ CVEDataNumberOfCVEs string `json:"CVE_data_numberOfCVEs,omitempty"`
+ CVEDataTimestamp string `json:"CVE_data_timestamp,omitempty"`
+ CVEDataType string `json:"CVE_data_type"`
+ CVEDataVersion string `json:"CVE_data_version"`
+ CVEItems []*NVDCVEFeedJSON10DefCVEItem `json:"CVE_Items"`
+}
diff --git a/server/vulnerabilities/nvd/tools/cvefeed/nvd/smartvercmp.go b/server/vulnerabilities/nvd/tools/cvefeed/nvd/smartvercmp.go
new file mode 100644
index 0000000000..c6a0927a1c
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cvefeed/nvd/smartvercmp.go
@@ -0,0 +1,98 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nvd
+
+import (
+ "strings"
+)
+
+// smartVerCmp compares stringified versions of software.
+// It tries to do the right thing for any type of versioning,
+// assuming v1 and v2 have the same version convension.
+// It will return meaningful result for "95SE" vs "98SP1" or for "16.3.2" vs. "3.7.0",
+// but not for "2000" vs "11.7".
+// Returns -1 if v1 < v2, 1 if v1 > v2 and 0 if v1 == v2.
+func smartVerCmp(v1, v2 string) int {
+ s1, s2 := v1, v2
+ for len(s1) > 0 && len(s2) > 0 {
+ num1, cmpTo1, skip1 := parseVerParts(s1)
+ num2, cmpTo2, skip2 := parseVerParts(s2)
+
+ ns1 := s1[:cmpTo1]
+ ns2 := s2[:cmpTo2]
+ diff := num1 - num2
+ switch {
+ case diff > 0: // ns1 has longer numeric part
+ ns2 = lpad(ns2, diff)
+ case diff < 0: // ns2 has longer numeric part
+ ns1 = lpad(ns1, -diff)
+ }
+
+ if cmp := strings.Compare(ns1, ns2); cmp != 0 {
+ return cmp
+ }
+
+ s1 = s1[skip1:]
+ s2 = s2[skip2:]
+ }
+ // everything is equal so far, the longest wins
+ if len(s1) > len(s2) {
+ return 1
+ }
+ if len(s2) > len(s1) {
+ return -1
+ }
+ return 0
+}
+
+// parseVerParts returns the length of consecutive run of digits in the beginning of the string,
+// the last non-separator chararcted (which should be compared), and index at which the version part (major, minor etc.) ends,
+// i.e. the position of the dot or end of the line.
+// E.g. parseVerParts("11b.4.16-New_Year_Edition") will return (2, 3, 4)
+func parseVerParts(v string) (int, int, int) {
+ var num int
+ for num = 0; num < len(v); num++ {
+ if v[num] < '0' || v[num] > '9' {
+ break
+ }
+ }
+ if num == len(v) {
+ return num, num, num
+ }
+ // Any punctuation separates the parts.
+ skip := strings.IndexFunc(v, func(b rune) bool {
+ // !"#$%&'()*+,-./ are dec 33 to 47, :;<=>?@ are dec 58 to 64, [\]^_` are dec 91 to 96 and {|}~ are dec 123 to 126.
+ // So, punctuation is in dec 33-126 range except 48-57, 65-90 and 97-122 gaps.
+ // This inverse logic allows for early short-circuting for most of the chars and shaves ~20ns in benchmarks.
+ return b >= '!' && b <= '~' &&
+ !(b > '/' && b < ':' ||
+ b > '@' && b < '[' ||
+ b > '`' && b < '{')
+ })
+ if skip == -1 {
+ return num, len(v), len(v)
+ }
+ return num, skip, skip + 1
+}
+
+// lpad pads s with n '0's
+func lpad(s string, n int) string {
+ var sb strings.Builder
+ for i := 0; i < n; i++ {
+ sb.WriteByte('0')
+ }
+ sb.WriteString(s)
+ return sb.String()
+}
diff --git a/server/vulnerabilities/nvd/tools/cvefeed/nvd/smartvercmp_test.go b/server/vulnerabilities/nvd/tools/cvefeed/nvd/smartvercmp_test.go
new file mode 100644
index 0000000000..8909a71f06
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cvefeed/nvd/smartvercmp_test.go
@@ -0,0 +1,74 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nvd
+
+import (
+ "fmt"
+ "testing"
+)
+
+func TestSmartVerCmp(t *testing.T) {
+ cases := []struct {
+ v1, v2 string
+ ret int
+ }{
+ {"5", "8", -1},
+ {"15", "3", 1},
+ {"4a", "4c", -1},
+ {"1.0", "1.0", 0},
+ {"1.0.1", "1.0", 1},
+ {"1.0.14", "1.0.4", 1},
+ {"95SE", "98SP1", -1},
+ {"16.0.0", "3.2.7", 1},
+ {"10.23", "10.21", 1},
+ {"64.0", "3.6.24", 1},
+ {"5-1.15.2", "5-1.16", -1},
+ {"5-appl_1.16.1", "5-1.0.1", -1}, // this is wrong, but seems to be impossible to account for
+ {"5-1.16", "5_1.0.6", 1},
+ {"5-6", "5-16", -1},
+ {"5-a1", "5a1", -1}, // meh, kind of makes sense
+ {"5-a1", "5.a1", 0},
+ {"1.4", "1.02", 1},
+ {"5.0", "08.0", -1},
+ {"10.0", "1.0", 1},
+ {"2023.02.13", "2023.2.13", 0},
+ }
+ for _, c := range cases {
+ t.Run(fmt.Sprintf("%q vs %q", c.v1, c.v2), func(t *testing.T) {
+ if ret := smartVerCmp(c.v1, c.v2); ret != c.ret {
+ t.Fatalf("expected %d, got %d", c.ret, ret)
+ }
+ })
+ }
+}
+
+func BenchmarkSmartVerCmp(b *testing.B) {
+ cases := []struct {
+ v1, v2 string
+ }{
+ {"1.0", "1.0"},
+ {"1.0.1", "1.0"},
+ {"1.0.14", "1.0.4"},
+ {"95SE", "98SP1"},
+ {"16.0.0", "3.2.7"},
+ {"10.23", "10.21"},
+ }
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ for _, c := range cases {
+ smartVerCmp(c.v1, c.v2)
+ }
+ }
+}
diff --git a/server/vulnerabilities/nvd/tools/cvefeed/stats.go b/server/vulnerabilities/nvd/tools/cvefeed/stats.go
new file mode 100644
index 0000000000..56c0920261
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cvefeed/stats.go
@@ -0,0 +1,154 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cvefeed
+
+import (
+ "fmt"
+ "sort"
+ "strings"
+ "sync"
+
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/cvefeed/nvd"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/cvefeed/nvd/schema"
+)
+
+var cpeParts = map[string]string{
+ "a": "application",
+ "h": "hardware",
+ "o": "operating system",
+}
+
+type stack struct {
+ items []string
+ rwLock sync.RWMutex
+}
+
+func (s *stack) push(item string) {
+ s.rwLock.Lock()
+ defer s.rwLock.Unlock()
+ if s.items == nil {
+ s.items = []string{}
+ }
+ s.items = append(s.items, item)
+}
+
+func (s *stack) pop() (string, bool) {
+ if s.isEmpty() {
+ return "", false
+ }
+ s.rwLock.Lock()
+ defer s.rwLock.Unlock()
+ item := s.items[len(s.items)-1]
+ s.items = s.items[0 : len(s.items)-1]
+ return item, true
+}
+
+func (s *stack) isEmpty() bool {
+ s.rwLock.Lock()
+ defer s.rwLock.Unlock()
+ return len(s.items) == 0
+}
+
+// Stats contains the stats information of a NVD JSON feed
+type Stats struct {
+ totalCVEs int64
+ totalRules int64
+ totalRulesWithAND int64
+ operatorANDs map[string]int64
+}
+
+// NewStats creates a new Stats object
+func NewStats() *Stats {
+ s := Stats{}
+ s.Reset()
+ return &s
+}
+
+// Reset clears out a Stats object
+func (s *Stats) Reset() {
+ s.totalCVEs = 0
+ s.totalRules = 0
+ s.totalRulesWithAND = 0
+ s.operatorANDs = make(map[string]int64)
+}
+
+// ReportOperatorAND prints the stats of operator AND
+func (s *Stats) ReportOperatorAND() {
+ if s.totalRulesWithAND <= 0 {
+ fmt.Println("No rules found with AND operator.")
+ return
+ }
+ keys := make([]string, 0, len(s.operatorANDs))
+ for key := range s.operatorANDs {
+ keys = append(keys, key)
+ }
+ sort.Slice(keys, func(i, j int) bool {
+ return s.operatorANDs[keys[i]] > s.operatorANDs[keys[j]]
+ })
+ fmt.Printf("Total rules with AND operator: %0.2f%%\n", percentage(s.totalRulesWithAND, s.totalRules))
+ for _, key := range keys {
+ fmt.Printf("%05.2f%%: %s\n", percentage(s.operatorANDs[key], s.totalRulesWithAND), key)
+ }
+}
+
+// Gather feeds a Stats object by gathering stats from a NVD JSON feed dictionary
+func (s *Stats) Gather(dict Dictionary) {
+ for key := range dict {
+ s.totalCVEs++
+ schema := dict[key].(*nvd.Vuln).Schema()
+ for _, node := range schema.Configurations.Nodes {
+ s.totalRules++
+ rule := flattenRule(node, &stack{})
+ if strings.Contains(rule, "AND") {
+ s.totalRulesWithAND++
+ s.operatorANDs[rule]++
+ }
+ }
+ }
+}
+
+func flattenRule(node *schema.NVDCVEFeedJSON10DefNode, operators *stack) string {
+ cpePart := ""
+ operators.push(node.Operator)
+ switch {
+ case len(node.Children) > 0:
+ outputs := []string{}
+ for _, c := range node.Children {
+ outputs = append(outputs, flattenRule(c, operators))
+ }
+ operator, _ := operators.pop()
+ return fmt.Sprintf("(%s)", strings.Join(outputs, fmt.Sprintf(" %s ", operator)))
+ case len(node.CPEMatch) > 0:
+ for _, cpeMatch := range node.CPEMatch {
+ cpeItems := strings.Split(cpeMatch.Cpe23Uri, ":")
+ if len(cpeItems) > 2 {
+ part := cpeItems[2]
+ if _, ok := cpeParts[part]; ok && !strings.Contains(cpePart, part) {
+ cpePart += part
+ }
+ }
+ }
+ operator, _ := operators.pop()
+ if len(cpePart) > 1 {
+ return fmt.Sprintf("(%s)", strings.Join(strings.Split(cpePart, ""), fmt.Sprintf(" %s ", operator)))
+ }
+ }
+ return cpePart
+}
+
+func percentage(partial, total int64) (delta float64) {
+ delta = (float64(partial) / float64(total)) * 100
+ return
+}
diff --git a/server/vulnerabilities/nvd/tools/cvefeed/stats_test.go b/server/vulnerabilities/nvd/tools/cvefeed/stats_test.go
new file mode 100644
index 0000000000..be98bcca94
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cvefeed/stats_test.go
@@ -0,0 +1,200 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cvefeed
+
+import (
+ "io/ioutil"
+ "os"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+var testFeed = `
+{
+ "CVE_Items": [
+ {
+ "cve": {
+ "affects": null,
+ "CVE_data_meta": {
+ "ASSIGNER": "cve@mitre.org",
+ "ID": "CVE-2020-1111"
+ },
+ "data_format": "MITRE",
+ "data_type": "CVE",
+ "data_version": "4.0",
+ "description": {
+ "description_data": [
+ {
+ "lang": "en",
+ "value": ""
+ }
+ ]
+ },
+ "problemtype": {
+ "problemtype_data": [
+ {
+ "description": [
+ {
+ "lang": "en",
+ "value": "CWE-20"
+ }
+ ]
+ }
+ ]
+ },
+ "references": {
+ "reference_data": [
+ {
+ "name": "",
+ "refsource": "",
+ "tags": [
+ "Vendor Advisory"
+ ],
+ "url": ""
+ },
+ {
+ "name": "test",
+ "refsource": "MISC",
+ "url": ""
+ }
+ ]
+ }
+ },
+ "configurations": {
+ "CVE_data_version": "4.0",
+ "nodes": [
+ {
+ "children": [
+ {
+ "cpe_match": [
+ {
+ "cpe23Uri": "cpe:2.3:a:test:test:-:*:*:*:*:*:*:*",
+ "vulnerable": true
+ }
+ ],
+ "operator": "OR"
+ },
+ {
+ "cpe_match": [
+ {
+ "cpe23Uri": "cpe:2.3:h:test:test:-:*:*:*:*:*:*:*",
+ "vulnerable": false
+ }
+ ],
+ "operator": "OR"
+ }
+ ],
+ "operator": "AND"
+ },
+ {
+ "children": [
+ {
+ "cpe_match": [
+ {
+ "cpe23Uri": "cpe:2.3:a:test:test:-:*:*:*:*:*:*:*",
+ "vulnerable": true
+ }
+ ],
+ "operator": "OR"
+ },
+ {
+ "cpe_match": [
+ {
+ "cpe23Uri": "cpe:2.3:h:test:test:-:*:*:*:*:*:*:*",
+ "vulnerable": false
+ },
+ {
+ "cpe23Uri": "cpe:2.3:h:test:test:-:*:*:*:*:*:*:*",
+ "vulnerable": false
+ }
+ ],
+ "operator": "OR"
+ }
+ ],
+ "operator": "AND"
+ },
+ {
+ "children": [
+ {
+ "cpe_match": [
+ {
+ "cpe23Uri": "cpe:2.3:o:test:test:-:*:*:*:*:*:*:*",
+ "vulnerable": true
+ }
+ ],
+ "operator": "OR"
+ },
+ {
+ "cpe_match": [
+ {
+ "cpe23Uri": "cpe:2.3:a:test:test:-:*:*:*:*:*:*:*",
+ "vulnerable": false
+ },
+ {
+ "cpe23Uri": "cpe:2.3:h:test:test:-:*:*:*:*:*:*:*",
+ "vulnerable": false
+ }
+ ],
+ "operator": "OR"
+ }
+ ],
+ "operator": "AND"
+ },
+ {
+ "cpe_match": [
+ {
+ "cpe23Uri": "cpe:2.3:a:test:test:-:*:*:*:*:*:*:*",
+ "vulnerable": true
+ }
+ ],
+ "operator": "OR"
+ }
+ ]
+ }
+ }
+ ]
+}
+`
+
+func TestStats(t *testing.T) {
+ file, err := ioutil.TempFile("/tmp", "test_nvd.json")
+ assert.Nil(t, err, "Unexpected error occurred when creating a temp file")
+ defer func() {
+ assert.Nil(t, file.Close(), "Unexpected error occurred when closing a temp file")
+ assert.Nil(t, os.Remove(file.Name()), "Unexpected error occurred when removing a temp file")
+ }()
+ _, err = file.WriteString(testFeed)
+ assert.Nil(t, err, "Unexpected error occurred when writing to a temp file")
+ feedDict, err := LoadJSONDictionary(file.Name())
+ assert.Nil(t, err, "Unexpected error occurred when loading a test NVD JSON feed file")
+
+ stats := NewStats()
+ stats.Gather(feedDict)
+
+ orgStdOut := os.Stdout
+ r, w, _ := os.Pipe()
+ os.Stdout = w
+ stats.ReportOperatorAND()
+ w.Close()
+ output, _ := ioutil.ReadAll(r)
+ os.Stdout = orgStdOut
+
+ expectedOutput := `Total rules with AND operator: 75.00%
+66.67%: (a AND h)
+33.33%: (o AND (a OR h))
+`
+ assert.Equal(t, expectedOutput, string(output))
+}
diff --git a/server/vulnerabilities/nvd/tools/cvefeed/vuln.go b/server/vulnerabilities/nvd/tools/cvefeed/vuln.go
new file mode 100644
index 0000000000..1316b5edeb
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/cvefeed/vuln.go
@@ -0,0 +1,80 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cvefeed
+
+import (
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/wfn"
+)
+
+// Vuln is a vulnerability interface
+type Vuln interface {
+ // vulnerability should also be able to match attributes
+ wfn.Matcher
+ // ID returns the vulnerability ID
+ ID() string
+ // CVEs returns all CVEs it includes/references
+ CVEs() []string
+ // CWEs returns all CWEs for this vulnerability
+ CWEs() []string
+ // CVSSv2BaseScore returns CVSS v2 base score
+ CVSSv2BaseScore() float64
+ // CVSSv2BaseScore returns CVSS v2 vector
+ CVSSv2Vector() string
+ // CVSSv2BaseScore returns CVSS v3 base score
+ CVSSv3BaseScore() float64
+ // CVSSv2BaseScore returns CVSS v3 vector
+ CVSSv3Vector() string
+}
+
+// MergeVuln combines two Vulns:
+// resulted Vuln inherits all mutually exclusive methods (e.g. ID()) from Vuln x;
+// functions returning CVEs and CWEs return distinct(union(x,y))
+// the returned vuln matches attributes if x matches AND y doesn't
+func OverrideVuln(v, override Vuln) Vuln {
+ return &overriden{
+ Vuln: v,
+ matcher: &andMatcher{v, wfn.DontMatch(override)},
+ }
+}
+
+type overriden struct {
+ Vuln
+ matcher wfn.Matcher
+}
+
+// Match is a part of the wfn.Matcher interface
+func (v *overriden) Match(attrs []*wfn.Attributes, requireVersion bool) []*wfn.Attributes {
+ return v.matcher.Match(attrs, requireVersion)
+}
+
+// Attrs is a part of the wfn.Matcher interface
+func (v *overriden) Config() []*wfn.Attributes {
+ return v.matcher.Config()
+}
+
+// matches are the ones matched by both
+type andMatcher struct {
+ m1, m2 wfn.Matcher
+}
+
+// Match is a part of the wfn.Matcher interface
+func (m *andMatcher) Match(attrs []*wfn.Attributes, requireVersion bool) []*wfn.Attributes {
+ return m.m2.Match(m.m1.Match(attrs, requireVersion), requireVersion)
+}
+
+// Attrs is a part of the wfn.Matcher interface
+func (m *andMatcher) Config() []*wfn.Attributes {
+ return append(m.m1.Config(), m.m2.Config()...)
+}
diff --git a/server/vulnerabilities/nvd/tools/nvdtools.spec b/server/vulnerabilities/nvd/tools/nvdtools.spec
new file mode 100644
index 0000000000..2d08ba12ff
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/nvdtools.spec
@@ -0,0 +1,28 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+
+Name: nvdtools
+Summary: A collection of tools for working with National Vulnerability Database feeds.
+Version: %{_version}
+Release: 1
+License: Apache License 2.0
+URL: https://github.com/facebookincubator/nvdtools
+Source0: %{name}-%{version}.tar.gz
+
+%description
+A set of tools to work with the feeds (vulnerabilities, CPE dictionary etc.) distributed by National Vulnerability Database (NVD).
+
+%prep
+%setup -q
+
+%build
+make GOFLAGS="-ldflags=-linkmode=external"
+
+%install
+make install DESTDIR=$RPM_BUILD_ROOT
+
+%files
+%license LICENSE
+%{_bindir}/*
+/usr/share/doc/nvdtools
+
+%changelog
diff --git a/server/vulnerabilities/nvd/tools/providers/lib/client/client.go b/server/vulnerabilities/nvd/tools/providers/lib/client/client.go
new file mode 100644
index 0000000000..f70487ab3c
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/providers/lib/client/client.go
@@ -0,0 +1,66 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+)
+
+// Client is an interface used for making http requests
+type Client interface {
+ Do(req *http.Request) (*http.Response, error)
+ Get(url string) (*http.Response, error)
+}
+
+// Default returns the default http client to use
+func Default() Client {
+ return http.DefaultClient
+}
+
+// Get will create a GET request with given headers and call Do on the client
+func Get(ctx context.Context, c Client, url string, header http.Header) (*http.Response, error) {
+ req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+ if err != nil {
+ return nil, fmt.Errorf("cannot create http get request: %v", err)
+ }
+ req.Header = header
+
+ id := traceRequestStart(req)
+ if debug.failRequestNum == id {
+ return nil, &Err{
+ Code: 503,
+ Status: "Service Unavailable",
+ Body: "Request cancelled by debug feature",
+ }
+ }
+ resp, err := c.Do(req)
+ traceRequestEnd(id, resp)
+
+ return resp, err
+}
+
+// Err encapsulates stuff from the http.Response
+type Err struct {
+ Code int
+ Status string
+ Body string
+}
+
+// Error is a part of the error interface
+func (e *Err) Error() string {
+ return fmt.Sprintf("http error %s:\n %q", e.Status, e.Body)
+}
diff --git a/server/vulnerabilities/nvd/tools/providers/lib/client/debug.go b/server/vulnerabilities/nvd/tools/providers/lib/client/debug.go
new file mode 100644
index 0000000000..c00fa78a0b
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/providers/lib/client/debug.go
@@ -0,0 +1,119 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+ "fmt"
+ "net/http"
+ "net/http/httputil"
+ "os"
+ "strconv"
+ "sync/atomic"
+
+ "github.com/facebookincubator/flog"
+)
+
+var debug struct {
+ // Print HTTP requests and responses to stderr.
+ traceRequests bool
+ // Print the bodies of HTTP requests and responses to stderr.
+ traceRequestBodies bool
+ // When tools issue concurrent GET requests, the normal behaviour is to
+ // cancel pending requests as soon as one request fails. This option
+ // restores the old behaviour or executing the remaning requests anyway.
+ continueDownloading bool
+ // When set to a number n, the n th HTTP request will fail.
+ failRequestNum uint64
+
+ requestNum uint64
+}
+
+func getBool(varName string) bool {
+ v, _ := strconv.ParseBool(os.Getenv(varName))
+ return v
+}
+
+func getUint(varName string, defaultValue uint64) uint64 {
+ v, err := strconv.ParseUint(os.Getenv(varName), 10, 64)
+ if err != nil {
+ return defaultValue
+ }
+ return v
+}
+
+func init() {
+ debug.traceRequests = getBool("NVD_TRACE_REQUESTS")
+ debug.traceRequestBodies = getBool("NVD_TRACE_REQUEST_BODIES")
+ debug.continueDownloading = getBool("NVD_CONTINUE_DOWNLOADING")
+ debug.failRequestNum = getUint("NVD_FAIL_REQUEST", 0)
+}
+
+func obfuscateHeaders(req *http.Request) *http.Request {
+ authHeaders := []string{
+ "Authorization",
+ // fireeye
+ "X-Auth",
+ "X-Auth-Hash",
+ // idefense
+ "Auth-Token",
+ }
+
+ headers := req.Header.Clone()
+ for _, header := range authHeaders {
+ if headers.Get(header) == "" {
+ continue
+ }
+ headers.Set(header, "")
+ }
+
+ // A shallow copy is enough for this usage.
+ newReq := *req
+ newReq.Header = headers
+ return &newReq
+}
+
+func traceRequestStart(req *http.Request) uint64 {
+ id := atomic.AddUint64(&debug.requestNum, 1)
+ if !debug.traceRequests {
+ return id
+ }
+ data, _ := httputil.DumpRequest(obfuscateHeaders(req), debug.traceRequestBodies)
+ fmt.Fprintf(os.Stderr, "Req %d: %s", id, string(data))
+ return id
+}
+
+func traceRequestEnd(id uint64, resp *http.Response) {
+ if !debug.traceRequests {
+ return
+ }
+ if resp == nil {
+ return
+ }
+ data, _ := httputil.DumpResponse(resp, debug.traceRequestBodies)
+ fmt.Fprintf(os.Stderr, "Req %d: %s", id, string(data))
+}
+
+// StopOrContinue can help controlling the behaviour of concurrent GET requests
+// when using an errgroup and encountering an error. Depending on the
+// NVD_CONTINUE_DOWNLOADING env variable, this function will return the passed
+// error (when we want to stop pending requests) or just log the error (when we
+// want the pending requests to continue being processed).
+func StopOrContinue(err error) error {
+ if debug.continueDownloading {
+ flog.Errorln(err)
+ return nil
+ }
+ return err
+}
diff --git a/server/vulnerabilities/nvd/tools/providers/lib/rate/rate.go b/server/vulnerabilities/nvd/tools/providers/lib/rate/rate.go
new file mode 100644
index 0000000000..41f0551a74
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/providers/lib/rate/rate.go
@@ -0,0 +1,51 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rate
+
+import (
+ "time"
+)
+
+// Limiter provides only one function: Allow. it blocks until routine can proceed
+type Limiter interface {
+ Allow()
+}
+
+type token struct{}
+
+type limiter struct {
+ tokens chan token
+}
+
+// BurstyLimiter will create a limiter which allows bursts of maximum requestsPerPeriod
+// and otherwise allows requests with period/requestsPerPeriod gap in between
+func BurstyLimiter(period time.Duration, requestsPerPeriod int) Limiter {
+ l := &limiter{
+ tokens: make(chan token, requestsPerPeriod),
+ }
+
+ // start filling indefinitely
+ go func() {
+ for range time.Tick(period / time.Duration(requestsPerPeriod)) {
+ l.tokens <- token{}
+ }
+ }()
+
+ return l
+}
+
+func (l *limiter) Allow() {
+ <-l.tokens
+}
diff --git a/server/vulnerabilities/nvd/tools/providers/nvd/cpe.go b/server/vulnerabilities/nvd/tools/providers/nvd/cpe.go
new file mode 100644
index 0000000000..def9db3c06
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/providers/nvd/cpe.go
@@ -0,0 +1,224 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nvd
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "net/url"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "github.com/facebookincubator/flog"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/providers/lib/client"
+)
+
+// CPE defines the CPE data feed for synchronization.
+type CPE int
+
+// Supported CPE feeds.
+const (
+ cpe23xmlGz CPE = iota // CPE database in XML 2.3 format, gzip compressed.
+ cpe23xmlZip // CPE database in XML 2.3 format, zip compressed.
+ cpe22xmlGz // CPE database in XML 2.2 format, gzip compressed.
+ cpe22xmlZip // CPE database in XML 2.2 format, zip compressed.
+)
+
+// SupportedCPE contains all supported CPE data feeds indexed by name.
+var SupportedCPE = map[string]CPE{
+ "cpe-2.2.xml.gz": cpe22xmlGz,
+ "cpe-2.2.xml.zip": cpe22xmlZip,
+ "cpe-2.3.xml.gz": cpe23xmlGz,
+ "cpe-2.3.xml.zip": cpe23xmlZip,
+}
+
+// Set implements the flag.Value interface.
+func (c *CPE) Set(v string) error {
+ feed, exists := SupportedCPE[v]
+ if !exists {
+ return fmt.Errorf("unsupported CPE feed: %q", v)
+ }
+ *c = feed
+ return nil
+}
+
+// String implements the fmt.Stringer interface.
+func (c CPE) String() string {
+ return "cpe-" + c.version() + ".xml." + c.compression()
+}
+
+// Help returns the CPE flag help.
+func (c CPE) Help() string {
+ opts := make([]string, 0, len(SupportedCPE))
+ for k := range SupportedCPE {
+ opts = append(opts, k)
+ }
+ sort.Strings(opts)
+ return fmt.Sprintf(
+ "CPE feed to sync (default: %s)\navailable:\n%s",
+ c, strings.Join(opts, "\n"),
+ )
+}
+
+// compression returns the data feed compression: gz or zip.
+func (c CPE) compression() string {
+ switch c {
+ case cpe22xmlGz, cpe23xmlGz:
+ return "gz"
+ case cpe22xmlZip, cpe23xmlZip:
+ return "zip"
+ default:
+ panic("unsupported CPE compression")
+ }
+}
+
+// version returns the data feed version.
+func (c CPE) version() string {
+ switch c {
+ case cpe22xmlGz, cpe22xmlZip:
+ return "2.2"
+ case cpe23xmlGz, cpe23xmlZip:
+ return "2.3"
+ default:
+ panic("unsupported CPE version")
+ }
+}
+
+// Sync synchronizes the CPE feed to a local directory.
+func (c CPE) Sync(ctx context.Context, src SourceConfig, localdir string) error {
+ basename := "official-cpe-dictionary_v" + c.version()
+ cf := cpeFile{
+ CPE: c,
+ EtagFile: basename + ".etag",
+ DataFile: basename + ".xml." + c.compression(),
+ }
+ return cf.Sync(ctx, src, localdir)
+}
+
+type cpeFile struct {
+ CPE
+ EtagFile string
+ DataFile string
+}
+
+func (cf cpeFile) baseURL(src SourceConfig) string {
+ u := url.URL{
+ Scheme: src.Scheme,
+ Host: src.Host,
+ Path: src.CPEFeedPath,
+ }
+ baseURL := u.String()
+ if !strings.HasSuffix(baseURL, "/") {
+ baseURL += "/"
+ }
+ return baseURL
+}
+
+func (cf cpeFile) Sync(ctx context.Context, src SourceConfig, localdir string) error {
+ baseURL := cf.baseURL(src)
+ sourceURL := baseURL + cf.DataFile
+ needsUpdate, err := cf.needsUpdate(ctx, sourceURL, localdir)
+ if err != nil {
+ return err
+ }
+ if !needsUpdate {
+ return nil
+ }
+ etag, tempDataFilename, err := cf.download(ctx, sourceURL)
+ if err != nil {
+ return err
+ }
+ defer os.Remove(tempDataFilename)
+
+ // write etag file
+ etagFilename := filepath.Join(localdir, cf.EtagFile)
+ err = ioutil.WriteFile(etagFilename, []byte(etag), 0o644)
+ if err != nil {
+ return err
+ }
+
+ // write data file
+ dataFilename := filepath.Join(localdir, cf.DataFile)
+ bakDataFilename := dataFilename + ".bak"
+ _ = xRename(dataFilename, bakDataFilename)
+ if err = xRename(tempDataFilename, dataFilename); err != nil {
+ _ = xRename(bakDataFilename, dataFilename)
+ return err
+ }
+ os.Remove(bakDataFilename)
+ return nil
+}
+
+func (cf cpeFile) needsUpdate(ctx context.Context, targetURL, localdir string) (bool, error) {
+ flog.V(1).Infof("checking etag for %q", targetURL)
+ req, err := httpNewRequestContext(ctx, "HEAD", targetURL)
+ if err != nil {
+ return false, err
+ }
+ resp, err := client.Default().Do(req)
+ if err != nil {
+ return false, err
+ }
+ defer resp.Body.Close()
+ if err = httpResponseNotOK(resp); err != nil {
+ return false, err
+ }
+ remoteEtag := resp.Header.Get("Etag")
+ if remoteEtag == "" {
+ return false, fmt.Errorf("server not returning etag header for %q", targetURL)
+ }
+ etagBytes, err := ioutil.ReadFile(filepath.Join(localdir, cf.EtagFile))
+ if err != nil {
+ flog.V(1).Infof("etag file %q for not exist in %q, needs sync", cf.EtagFile, localdir)
+ return true, nil
+ }
+ localEtag := string(etagBytes)
+ if localEtag != remoteEtag {
+ flog.V(1).Infof("data file %q needs update in %q: hash mismatch %q != %q", cf.DataFile, localdir, localEtag, remoteEtag)
+ return true, nil
+ }
+ return false, nil
+}
+
+// download file from targetURL, returns etag and path to local file.
+func (cf cpeFile) download(ctx context.Context, targetURL string) (string, string, error) {
+ flog.V(1).Infof("downloading data file %q", targetURL)
+ req, err := http.NewRequest("GET", targetURL, nil)
+ if err != nil {
+ return "", "", err
+ }
+ resp, err := client.Default().Do(req.WithContext(ctx))
+ if err != nil {
+ return "", "", err
+ }
+ defer resp.Body.Close()
+ if err = httpResponseNotOK(resp); err != nil {
+ return "", "", err
+ }
+ dataFile, err := ioutil.TempFile("", "nvdsync-data-")
+ if err != nil {
+ return "", "", err
+ }
+ _, err = io.Copy(dataFile, resp.Body)
+ if err != nil {
+ return "", "", err
+ }
+ return resp.Header.Get("Etag"), dataFile.Name(), nil
+}
diff --git a/server/vulnerabilities/nvd/tools/providers/nvd/cpe_test.go b/server/vulnerabilities/nvd/tools/providers/nvd/cpe_test.go
new file mode 100644
index 0000000000..68de82ef3a
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/providers/nvd/cpe_test.go
@@ -0,0 +1,61 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nvd
+
+import (
+ "context"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "os"
+ "testing"
+)
+
+func TestCPE(t *testing.T) {
+ td, err := ioutil.TempDir("", "nvdsync-")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(td)
+
+ handler := &cpeTestServer{}
+ ts, src := httptestNewServer(handler)
+ defer ts.Close()
+
+ cases := make([]CPE, 0, len(SupportedCPE))
+ for _, cve := range SupportedCPE {
+ cases = append(cases, cve)
+ }
+
+ for _, cpe := range cases {
+ label := []string{"CreateSync", "UseExistingSync"}
+ for i := 0; i < 2; i++ {
+ info := fmt.Sprintf("%s/%s", label[i], cpe)
+ t.Run(info, func(t *testing.T) {
+ err = cpe.Sync(context.Background(), src, td)
+ if err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+ }
+}
+
+type cpeTestServer struct{}
+
+func (ts cpeTestServer) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Etag", "foobar")
+ fmt.Fprintf(w, "hello, world")
+}
diff --git a/server/vulnerabilities/nvd/tools/providers/nvd/cve.go b/server/vulnerabilities/nvd/tools/providers/nvd/cve.go
new file mode 100644
index 0000000000..2392f0b1e6
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/providers/nvd/cve.go
@@ -0,0 +1,586 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nvd
+
+import (
+ "archive/zip"
+ "bytes"
+ "compress/gzip"
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/url"
+ "os"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
+ "text/template"
+ "time"
+
+ "github.com/facebookincubator/flog"
+ "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/providers/lib/client"
+)
+
+// CVE defines the CVE data feed for synchronization.
+type CVE int
+
+// Supported CVE feeds.
+const (
+ cve20xmlGz CVE = iota // CVE database in XML 2.0 format, gzip compressed.
+ cve20xmlZip // CVE database in XML 2.0 format, zip compressed.
+ cve12xmlGz // CVE database in XML 1.2 format, gzip compressed.
+ cve12xmlZip // CVE database in XML 1.2 format, zip compressed.
+ cve10jsonGz // CVE database in JSON 1.0 format, gzip compressed.
+ cve10jsonZip // CVE database in JSON 1.0 format, zip compressed.
+ cve11jsonGz // CVE database in JSON 1.1 format, gzip compressed.
+ cve11jsonZip // CVE database in JSON 1.1 format, zip compressed.
+)
+
+// SupportedCVE contains all supported CVE feeds indexed by name.
+var SupportedCVE = map[string]CVE{
+ "cve-1.2.xml.gz": cve12xmlGz,
+ "cve-1.2.xml.zip": cve12xmlZip,
+ "cve-2.0.xml.gz": cve20xmlGz,
+ "cve-2.0.xml.zip": cve20xmlZip,
+ "cve-1.0.json.gz": cve10jsonGz,
+ "cve-1.0.json.zip": cve10jsonZip,
+ "cve-1.1.json.gz": cve11jsonGz,
+ "cve-1.1.json.zip": cve11jsonZip,
+}
+
+// Set implements the flag.Value interface.
+func (c *CVE) Set(v string) error {
+ feed, exists := SupportedCVE[v]
+ if !exists {
+ return fmt.Errorf("unsupported CVE feed: %q", v)
+ }
+ *c = feed
+ return nil
+}
+
+// String implements the fmt.Stringer interface.
+func (c CVE) String() string {
+ return "cve-" + c.version() + "." + c.encoding() + "." + c.compression()
+}
+
+// Help returns the CVE flag help.
+func (c CVE) Help() string {
+ opts := make([]string, 0, len(SupportedCVE))
+ for k := range SupportedCVE {
+ opts = append(opts, k)
+ }
+ sort.Strings(opts)
+ return fmt.Sprintf(
+ "CVE feed to sync (default: %s)\navailable:\n%s",
+ c, strings.Join(opts, "\n"),
+ )
+}
+
+// encoding returns the data feed encoding: xml or json.
+func (c CVE) encoding() string {
+ switch c {
+ case cve12xmlGz, cve12xmlZip, cve20xmlGz, cve20xmlZip:
+ return "xml"
+ case cve10jsonGz, cve10jsonZip, cve11jsonGz, cve11jsonZip:
+ return "json"
+ default:
+ panic("unsupported CVE encoding")
+ }
+}
+
+// compression returns the data feed compression: gz or zip.
+func (c CVE) compression() string {
+ switch c {
+ case cve10jsonGz, cve11jsonGz, cve12xmlGz, cve20xmlGz:
+ return "gz"
+ case cve10jsonZip, cve11jsonZip, cve12xmlZip, cve20xmlZip:
+ return "zip"
+ default:
+ panic("unsupported CVE compression")
+ }
+}
+
+// version returns the data feed version.
+func (c CVE) version() string {
+ switch c {
+ case cve12xmlGz, cve12xmlZip:
+ return "1.2"
+ case cve20xmlGz, cve20xmlZip:
+ return "2.0"
+ case cve10jsonGz, cve10jsonZip:
+ return "1.0"
+ case cve11jsonGz, cve11jsonZip:
+ return "1.1"
+ default:
+ panic("unsupported CVE version")
+ }
+}
+
+// Sync synchronizes the CVE feed to a local directory.
+func (c CVE) Sync(ctx context.Context, src SourceConfig, localdir string) error {
+ var err error
+ files := cveFileList(c)
+ for _, f := range files {
+ if err = f.Sync(ctx, src, localdir); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func cveFileList(c CVE) []cveFile {
+ filefmt := func(version, suffix, encoding, compression string) string {
+ s := fmt.Sprintf("nvdcve-%s-%s.%s", version, suffix, encoding)
+ if compression != "" {
+ s += "." + compression
+ }
+ return s
+ }
+
+ // nvd data feeds start in 2002
+ const startingYear = 2002
+ currentYear := time.Now().Year()
+ if currentYear < startingYear {
+ panic("system date is in the past, cannot continue")
+ }
+
+ entries := (currentYear - startingYear) + 1
+ f := make([]cveFile, entries+2) // +recent +modified
+
+ version := c.version()
+ encoding := c.encoding()
+ compression := c.compression()
+
+ for i := 0; i < entries; i++ {
+ year := startingYear + i
+ suffix := strconv.Itoa(year)
+ f[i] = cveFile{
+ CVE: c,
+ MetaFile: filefmt(version, suffix, "meta", ""),
+ DataFile: filefmt(version, suffix, encoding, compression),
+ }
+ }
+
+ // recent
+ f[entries] = cveFile{
+ CVE: c,
+ MetaFile: filefmt(version, "recent", "meta", ""),
+ DataFile: filefmt(version, "recent", encoding, compression),
+ }
+
+ // modified
+ f[entries+1] = cveFile{
+ CVE: c,
+ MetaFile: filefmt(version, "modified", "meta", ""),
+ DataFile: filefmt(version, "modified", encoding, compression),
+ }
+
+ return f
+}
+
+type cveFile struct {
+ CVE
+ MetaFile string
+ DataFile string
+}
+
+func (cf cveFile) baseURL(src SourceConfig) (string, error) {
+ tmpl, err := template.New("path").Parse(src.CVEFeedPath)
+ if err != nil {
+ return "", err
+ }
+ b := bytes.Buffer{}
+ err = tmpl.Execute(&b, struct {
+ Encoding string
+ Version string
+ }{
+ Encoding: cf.encoding(),
+ Version: cf.version(),
+ })
+ if err != nil {
+ return "", err
+ }
+ u := url.URL{
+ Scheme: src.Scheme,
+ Host: src.Host,
+ Path: b.String(),
+ }
+ baseURL := u.String()
+ if !strings.HasSuffix(baseURL, "/") {
+ baseURL += "/"
+ }
+ return baseURL, nil
+}
+
+func (cf cveFile) Sync(ctx context.Context, src SourceConfig, localdir string) error {
+ baseURL, err := cf.baseURL(src)
+ if err != nil {
+ return err
+ }
+ remoteMetaURL := baseURL + cf.MetaFile
+ flog.V(1).Infof("checking meta file %q for updates to %q", cf.MetaFile, cf.DataFile)
+ remoteMeta, needsUpdate, err := cf.needsUpdate(ctx, remoteMetaURL, localdir)
+ if err != nil {
+ return err
+ }
+ if !needsUpdate {
+ return nil
+ }
+ remoteFileURL := baseURL + cf.DataFile
+ tempDataFilename, err := cf.downloadAndVerify(ctx, remoteMeta, remoteFileURL)
+ if err != nil {
+ return err
+ }
+ defer os.Remove(tempDataFilename)
+
+ // write metadata file
+ metaFilename := filepath.Join(localdir, cf.MetaFile)
+ err = remoteMeta.WriteFile(metaFilename)
+ if err != nil {
+ return err
+ }
+
+ // write data file
+ dataFilename := filepath.Join(localdir, cf.DataFile)
+ bakDataFilename := dataFilename + ".bak"
+ _ = xRename(dataFilename, bakDataFilename)
+ if err = xRename(tempDataFilename, dataFilename); err != nil {
+ _ = xRename(bakDataFilename, dataFilename)
+ return err
+ }
+ os.Remove(bakDataFilename)
+ return nil
+}
+
+func (cf cveFile) needsUpdate(ctx context.Context, remoteMetaURL, localdir string) (*metaFile, bool, error) {
+ flog.V(1).Infof("downloading meta file %q", remoteMetaURL)
+ remoteMeta, err := newMetaFromURL(ctx, remoteMetaURL)
+ if err != nil {
+ return nil, false, err
+ }
+ metaFilename := filepath.Join(localdir, cf.MetaFile)
+ if _, err := os.Stat(metaFilename); os.IsNotExist(err) {
+ flog.V(1).Infof("meta file %q does not exist in %q, needs sync", cf.MetaFile, localdir)
+ return &remoteMeta, true, nil
+ }
+ localMeta, err := newMetaFromFile(metaFilename)
+ if err != nil {
+ return nil, false, err
+ }
+ if !localMeta.Equal(remoteMeta) {
+ flog.V(1).Infof("data file %q needs update in %q: local%+v != remote%+v", cf.DataFile, localdir, localMeta, remoteMeta)
+ return &remoteMeta, true, nil
+ }
+ dataFilename := filepath.Join(localdir, cf.DataFile)
+ fi, err := os.Stat(dataFilename)
+ if err != nil {
+ if os.IsNotExist(err) {
+ flog.V(1).Infof("data file %q does not exist in %q, needs sync", cf.DataFile, localdir)
+ return &remoteMeta, true, nil
+ }
+ return nil, false, err
+ }
+ var sizeOK bool
+ var hashFunc func(filename string) (string, error)
+ switch cf.compression() {
+ case "gz":
+ sizeOK = fi.Size() == int64(localMeta.GzSize)
+ hashFunc = gunzipFileAndComputeSHA256
+ case "zip":
+ sizeOK = fi.Size() == int64(localMeta.ZipSize)
+ hashFunc = unzipFileAndComputeSHA256
+ }
+ if !sizeOK {
+ flog.V(1).Infof("data file %q needs update in %q: size mismatch", cf.DataFile, localdir)
+ return &remoteMeta, true, nil
+ }
+ hash, err := hashFunc(dataFilename)
+ if err != nil {
+ return nil, false, err
+ }
+ if hash != localMeta.SHA256 {
+ flog.V(1).Infof("data file %q needs update in %q: hash mismatch %q != %q", cf.DataFile, localdir, hash, localMeta.SHA256)
+ return &remoteMeta, true, nil
+ }
+ return &remoteMeta, false, nil
+}
+
+// downloadAndVerify downloads a remote file into a temporary local file, and performs checksum using size and hash from m.
+// Returns the path to the local file.
+func (cf cveFile) downloadAndVerify(ctx context.Context, m *metaFile, remoteFileURL string) (string, error) {
+ req, err := httpNewRequestContext(ctx, "GET", remoteFileURL)
+ if err != nil {
+ return "", err
+ }
+ flog.V(1).Infof("downloading data file %q", remoteFileURL)
+ resp, err := client.Default().Do(req)
+ if err != nil {
+ return "", err
+ }
+ defer resp.Body.Close()
+ if err = httpResponseNotOK(resp); err != nil {
+ return "", err
+ }
+ var wantSize int64
+ var hashFunc func(filename string) (string, error)
+ switch cf.compression() {
+ case "gz":
+ wantSize = int64(m.GzSize)
+ hashFunc = gunzipFileAndComputeSHA256
+ case "zip":
+ wantSize = int64(m.ZipSize)
+ hashFunc = unzipFileAndComputeSHA256
+ }
+ if resp.ContentLength != wantSize {
+ return "", fmt.Errorf(
+ "unexpected size for %q (%s): want %d, have %d",
+ remoteFileURL, resp.Status, wantSize, resp.ContentLength,
+ )
+ }
+ dataFile, err := ioutil.TempFile("", "nvdsync-data-")
+ if err != nil {
+ return "", err
+ }
+ _, err = io.Copy(dataFile, resp.Body)
+ dataFile.Close()
+ if err != nil {
+ return "", err
+ }
+ hash, err := hashFunc(dataFile.Name())
+ if err != nil {
+ defer os.Remove(dataFile.Name()) // TODO: delet?
+ return "", err
+ }
+ if hash != m.SHA256 {
+ defer os.Remove(dataFile.Name()) // TODO: delet?
+ return "", fmt.Errorf(
+ "unexpected hash for %q (%s): want %q, have %q",
+ remoteFileURL, resp.Status, m.SHA256, hash,
+ )
+ }
+ return dataFile.Name(), nil
+}
+
+// metaFile represents a .meta file from CVE data feeds.
+type metaFile struct {
+ LastModifiedDate time.Time
+ Size int
+ ZipSize int
+ GzSize int
+ SHA256 string
+}
+
+// Equal compares two meta files.
+func (m metaFile) Equal(other metaFile) bool {
+ switch {
+ case
+ !m.LastModifiedDate.Equal(other.LastModifiedDate),
+ m.Size != other.Size,
+ m.ZipSize != other.ZipSize,
+ m.GzSize != other.GzSize,
+ m.SHA256 != other.SHA256:
+ return false
+ }
+ return true
+}
+
+// WriteTo writes the contents of m to w.
+func (m metaFile) WriteTo(w io.Writer) (int64, error) {
+ lines := []string{
+ "lastModifiedDate:%s\r\n",
+ "size:%d\r\n",
+ "zipSize:%d\r\n",
+ "gzSize:%d\r\n",
+ "sha256:%s\r\n",
+ }
+ params := []interface{}{
+ m.LastModifiedDate.Format(time.RFC3339),
+ m.Size,
+ m.ZipSize,
+ m.GzSize,
+ strings.ToUpper(m.SHA256),
+ }
+ var total int64
+ for i := 0; i < len(lines); i++ {
+ n, err := fmt.Fprintf(w, lines[i], params[i])
+ if err != nil {
+ return 0, err
+ }
+ total += int64(n)
+ }
+ return total, nil
+}
+
+// WriteFile writes m to a file.
+func (m metaFile) WriteFile(name string) error {
+ f, err := ioutil.TempFile("", "nvdsync-meta-")
+ if err != nil {
+ return err
+ }
+ _, err = m.WriteTo(f)
+ f.Close()
+ if err != nil {
+ return err
+ }
+ bak := name + ".bak"
+ _ = xRename(name, bak)
+ if err = xRename(f.Name(), name); err != nil {
+ _ = xRename(bak, name)
+ return err
+ }
+ os.Remove(bak)
+ return err
+}
+
+// newMetaFile loads metadata from r.
+func newMetaFile(r io.Reader) (metaFile, error) {
+ m := metaFile{}
+ r = io.LimitReader(r, 16*1024)
+ b, err := ioutil.ReadAll(r)
+ if err != nil {
+ return m, err
+ }
+ lines := bytes.Split(b, []byte("\r\n"))
+ for i, line := range lines {
+ if len(line) == 0 {
+ break
+ }
+ lineno := i + 1
+ parts := bytes.SplitN(line, []byte(":"), 2)
+ if len(parts) != 2 {
+ return m, fmt.Errorf("line %d: expecting key:value not %q", lineno, string(line))
+ }
+ key := string(parts[0])
+ val := string(parts[1])
+ switch key {
+ case "lastModifiedDate":
+ t, err := time.Parse(time.RFC3339, val)
+ if err != nil {
+ return m, fmt.Errorf("line %d: expecting lastModifiedDate={RFC3339} not %q", lineno, string(line))
+ }
+ m.LastModifiedDate = t
+ case "size":
+ v, err := strconv.Atoi(val)
+ if err != nil {
+ return m, fmt.Errorf("line %d: expecting size={int} not %q", lineno, string(line))
+ }
+ m.Size = v
+ case "zipSize":
+ v, err := strconv.Atoi(val)
+ if err != nil {
+ return m, fmt.Errorf("line %d: expecting zipSize={int} not %q", lineno, string(line))
+ }
+ m.ZipSize = v
+ case "gzSize":
+ v, err := strconv.Atoi(val)
+ if err != nil {
+ return m, fmt.Errorf("line %d: expecting gzSize={int} not %q", lineno, string(line))
+ }
+ m.GzSize = v
+ case "sha256":
+ m.SHA256 = strings.ToUpper(val)
+ }
+ }
+ return m, nil
+}
+
+// newMetaFromURL loads metadata from a URL pointing to a .meta file.
+func newMetaFromURL(ctx context.Context, url string) (metaFile, error) {
+ m := metaFile{}
+ req, err := httpNewRequestContext(ctx, "GET", url)
+ if err != nil {
+ return m, err
+ }
+ resp, err := client.Default().Do(req)
+ if err != nil {
+ return m, err
+ }
+ defer resp.Body.Close()
+ if err = httpResponseNotOK(resp); err != nil {
+ return m, err
+ }
+ m, err = newMetaFile(resp.Body)
+ if err != nil {
+ return m, fmt.Errorf("malformed data in remote metadata %q: %v", url, err)
+ }
+ return m, nil
+}
+
+// newMetaFromFile loads metadata from a local .meta file.
+func newMetaFromFile(filename string) (metaFile, error) {
+ m := metaFile{}
+ f, err := os.Open(filename)
+ if err != nil {
+ return m, err
+ }
+ defer f.Close()
+ m, err = newMetaFile(f)
+ if err != nil {
+ return m, fmt.Errorf("malformed data in local metadata %q: %v", filename, err)
+ }
+ return m, nil
+}
+
+func computeSHA256(r io.Reader) (string, error) {
+ hasher := sha256.New()
+ _, err := io.Copy(hasher, r)
+ if err != nil {
+ return "", err
+ }
+ hash := hasher.Sum(nil)
+ return strings.ToUpper(hex.EncodeToString(hash)), nil
+}
+
+func gunzipAndComputeSHA256(r io.Reader) (string, error) {
+ f, err := gzip.NewReader(r)
+ if err != nil {
+ return "", err
+ }
+ defer f.Close()
+ return computeSHA256(f)
+}
+
+func gunzipFileAndComputeSHA256(filename string) (string, error) {
+ f, err := os.Open(filename)
+ if err != nil {
+ return "", err
+ }
+ defer f.Close()
+ return gunzipAndComputeSHA256(f)
+}
+
+func unzipFileAndComputeSHA256(filename string) (string, error) {
+ f, err := zip.OpenReader(filename)
+ if err != nil {
+ return "", err
+ }
+ defer f.Close()
+ if len(f.File) != 1 {
+ return "", fmt.Errorf(
+ "unexpected number of files in zip %q: want 1, have %d",
+ filename, len(f.File),
+ )
+ }
+ ff, err := f.File[0].Open()
+ if err != nil {
+ return "", err
+ }
+ defer ff.Close()
+ return computeSHA256(ff)
+}
diff --git a/server/vulnerabilities/nvd/tools/providers/nvd/cve_test.go b/server/vulnerabilities/nvd/tools/providers/nvd/cve_test.go
new file mode 100644
index 0000000000..a102897f32
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/providers/nvd/cve_test.go
@@ -0,0 +1,110 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nvd
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "os"
+ "strings"
+ "testing"
+)
+
+var (
+ cveGoldenMetaFile = strings.Join([]string{
+ "lastModifiedDate:2018-03-16T23:05:50-04:00",
+ "size:11",
+ "zipSize:169",
+ "gzSize:33",
+ "sha256:B94D27B9934D3E08A52E52D7DA7DABFAC484EFE37A5380EE9088F7ACE2EFCDE9",
+ }, "\r\n")
+
+ cveGoldenDataFileGz = []byte{
+ 0x1f, 0x8b, 0x08, 0x08, 0x42, 0x4d, 0xac, 0x5a, 0x02, 0x03, 0x66, 0x00,
+ 0xcb, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0x01,
+ 0x00, 0x85, 0x11, 0x4a, 0x0d, 0x0b, 0x00, 0x00, 0x00,
+ }
+
+ cveGoldenDataFileZip = []byte{
+ 0x50, 0x4b, 0x03, 0x04, 0x0a, 0x00, 0x02, 0x00, 0x00, 0x00, 0x6f, 0xb8,
+ 0x70, 0x4c, 0x85, 0x11, 0x4a, 0x0d, 0x0b, 0x00, 0x00, 0x00, 0x0b, 0x00,
+ 0x00, 0x00, 0x04, 0x00, 0x1c, 0x00, 0x74, 0x65, 0x73, 0x74, 0x55, 0x54,
+ 0x09, 0x00, 0x03, 0x42, 0x4d, 0xac, 0x5a, 0xeb, 0x80, 0xac, 0x5a, 0x75,
+ 0x78, 0x0b, 0x00, 0x01, 0x04, 0xad, 0x62, 0x2a, 0x64, 0x04, 0xba, 0x2d,
+ 0xd3, 0x6f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c,
+ 0x64, 0x50, 0x4b, 0x01, 0x02, 0x1e, 0x03, 0x0a, 0x00, 0x02, 0x00, 0x00,
+ 0x00, 0x6f, 0xb8, 0x70, 0x4c, 0x85, 0x11, 0x4a, 0x0d, 0x0b, 0x00, 0x00,
+ 0x00, 0x0b, 0x00, 0x00, 0x00, 0x04, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x01, 0x00, 0x00, 0x00, 0xa4, 0x81, 0x00, 0x00, 0x00, 0x00, 0x74,
+ 0x65, 0x73, 0x74, 0x55, 0x54, 0x05, 0x00, 0x03, 0x42, 0x4d, 0xac, 0x5a,
+ 0x75, 0x78, 0x0b, 0x00, 0x01, 0x04, 0xad, 0x62, 0x2a, 0x64, 0x04, 0xba,
+ 0x2d, 0xd3, 0x6f, 0x50, 0x4b, 0x05, 0x06, 0x00, 0x00, 0x00, 0x00, 0x01,
+ 0x00, 0x01, 0x00, 0x4a, 0x00, 0x00, 0x00, 0x49, 0x00, 0x00, 0x00, 0x00,
+ 0x00,
+ }
+)
+
+func TestCVE(t *testing.T) {
+ cases := make([]CVE, 0, len(SupportedCVE))
+ for _, cve := range SupportedCVE {
+ cases = append(cases, cve)
+ }
+
+ td, err := ioutil.TempDir("", "nvdsync-")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(td)
+
+ handler := &cveTestServer{}
+ ts, src := httptestNewServer(handler)
+ defer ts.Close()
+
+ for _, cve := range cases {
+ // run each test twice, one to create the mirror and another to compare
+ label := []string{"CreateSync", "UseExistingSync"}
+ for i := 0; i < 2; i++ {
+ info := fmt.Sprintf("%s/%s", label[i], cve)
+ t.Run(info, func(t *testing.T) {
+ handler.compression = cve.compression()
+ err = cve.Sync(context.Background(), src, td)
+ if err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+ }
+}
+
+type cveTestServer struct {
+ compression string
+}
+
+func (ts cveTestServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ if strings.HasSuffix(r.URL.Path, ".meta") {
+ _, _ = io.Copy(w, bytes.NewBufferString(cveGoldenMetaFile))
+ return
+ }
+ switch ts.compression {
+ case "gz":
+ _, _ = io.Copy(w, bytes.NewBuffer(cveGoldenDataFileGz))
+ case "zip":
+ _, _ = io.Copy(w, bytes.NewBuffer(cveGoldenDataFileZip))
+ }
+}
diff --git a/server/vulnerabilities/nvd/tools/providers/nvd/doc.go b/server/vulnerabilities/nvd/tools/providers/nvd/doc.go
new file mode 100644
index 0000000000..bd1e185832
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/providers/nvd/doc.go
@@ -0,0 +1,21 @@
+// Package datafeed provides NVD data feed synchronization for nvdsync.
+//
+// Designed for https://nvd.nist.gov/vuln/data-feeds.
+//
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package nvd
+
+// Version of nvdsync/datasync.
+const Version = "1.0"
diff --git a/server/vulnerabilities/nvd/tools/providers/nvd/e2e_test.go b/server/vulnerabilities/nvd/tools/providers/nvd/e2e_test.go
new file mode 100644
index 0000000000..15e96fe556
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/providers/nvd/e2e_test.go
@@ -0,0 +1,66 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nvd
+
+import (
+ "context"
+ "flag"
+ "io/ioutil"
+ "os"
+ "testing"
+ "time"
+)
+
+var (
+ e2eEnabled bool
+ e2eSource = NewSourceConfig()
+ e2eTimeout = 5 * time.Minute
+ e2eCVE = cve20xmlGz
+ e2eCPE = cpe23xmlGz
+)
+
+// test: go test -v -args -v=1 -logtostderr -e2e_enabled
+func init() {
+ flag.BoolVar(&e2eEnabled, "e2e_enabled", e2eEnabled, "enable end-to-end test")
+ flag.DurationVar(&e2eTimeout, "e2e_timeout", e2eTimeout, "timeout for end-to-end test")
+ flag.Var(&e2eCVE, "e2e_cve_feed", e2eCVE.Help())
+ flag.Var(&e2eCPE, "e2e_cpe_feed", e2eCPE.Help())
+ e2eSource.AddFlags(flag.CommandLine)
+}
+
+func TestEndToEnd(t *testing.T) {
+ if !e2eEnabled {
+ t.Skip("e2e tests not enabled")
+ }
+
+ d, err := ioutil.TempDir("", "nvdsync-")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(d)
+
+ ds := Sync{
+ Feeds: []Syncer{e2eCVE, e2eCPE},
+ Source: e2eSource,
+ LocalDir: d,
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), e2eTimeout)
+ defer cancel()
+
+ if err = ds.Do(ctx); err != nil {
+ t.Fatal(err)
+ }
+}
diff --git a/server/vulnerabilities/nvd/tools/providers/nvd/http.go b/server/vulnerabilities/nvd/tools/providers/nvd/http.go
new file mode 100644
index 0000000000..2f161103b2
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/providers/nvd/http.go
@@ -0,0 +1,63 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nvd
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "regexp"
+)
+
+var userAgent = "nvdsync-" + Version
+
+// http helpers
+
+func httpNewRequestContext(ctx context.Context, method, path string) (*http.Request, error) {
+ req, err := http.NewRequest(method, path, nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("User-Agent", UserAgent())
+ return req.WithContext(ctx), nil
+}
+
+func httpResponseNotOK(resp *http.Response) error {
+ if resp.StatusCode == http.StatusOK {
+ return nil
+ }
+ body, err := ioutil.ReadAll(io.LimitReader(resp.Body, 4*1024))
+ if err != nil {
+ return err
+ }
+ return fmt.Errorf("unexpected http response from %q (%q): %q",
+ resp.Request.URL.String(), resp.Status, string(body))
+}
+
+// SetUserAgent sets the value of User-Agent HTTP header for the client
+func SetUserAgent(ua string) error {
+ if !regexp.MustCompile("^[[:ascii:]]+$").MatchString(ua) {
+ return fmt.Errorf("non-ascii character in User-Agent header: %q", ua)
+ }
+ userAgent = ua
+ return nil
+}
+
+// UserAgent returns the value of User-Agent HTTP header used by the client
+func UserAgent() string {
+ return userAgent
+}
diff --git a/server/vulnerabilities/nvd/tools/providers/nvd/http_test.go b/server/vulnerabilities/nvd/tools/providers/nvd/http_test.go
new file mode 100644
index 0000000000..15c8aa5365
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/providers/nvd/http_test.go
@@ -0,0 +1,61 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nvd
+
+import (
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strings"
+ "testing"
+)
+
+// http test helpers
+
+func httptestNewServer(f http.Handler) (*httptest.Server, SourceConfig) {
+ ts := httptest.NewServer(f)
+
+ tsurl, _ := url.Parse(ts.URL)
+ src := SourceConfig{
+ Scheme: tsurl.Scheme,
+ Host: tsurl.Host,
+ CVEFeedPath: "/",
+ CPEFeedPath: "/",
+ }
+ return ts, src
+}
+
+func TestResponseNotOK(t *testing.T) {
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusServiceUnavailable)
+ fmt.Fprintf(w, "hello world")
+ }))
+ defer ts.Close()
+
+ resp, err := http.Get(ts.URL)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = httpResponseNotOK(resp)
+ if err == nil {
+ t.Fatal("unexpected response OK")
+ }
+
+ if !strings.Contains(err.Error(), "hello world") {
+ t.Fatalf("unexpected response: %q", err)
+ }
+}
diff --git a/server/vulnerabilities/nvd/tools/providers/nvd/src.go b/server/vulnerabilities/nvd/tools/providers/nvd/src.go
new file mode 100644
index 0000000000..4f2932fcf5
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/providers/nvd/src.go
@@ -0,0 +1,60 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nvd
+
+import (
+ "flag"
+ "os"
+ "reflect"
+)
+
+// SourceConfig is the configuration of the NVD data feed source.
+type SourceConfig struct {
+ Scheme string `envconfig:"NVDSYNC_SCHEME" default:"https"`
+ Host string `envconfig:"NVDSYNC_HOST" default:"nvd.nist.gov"`
+ CVEFeedPath string `envconfig:"NVDSYNC_CVE_FEED_PATH" default:"/feeds/{{.Encoding}}/cve/{{.Version}}/"`
+ CPEFeedPath string `envconfig:"NVDSYNC_CPE_FEED_PATH" default:"/feeds/xml/cpe/dictionary/"`
+}
+
+// NewSourceConfig creates and initializes a new SourceConfig with values from envconfig.
+func NewSourceConfig() *SourceConfig {
+ sc := &SourceConfig{}
+
+ valueFromStructTag := func(f reflect.StructField) string {
+ k := f.Tag.Get("envconfig")
+ if v := os.Getenv(k); v != "" {
+ return v
+ }
+ return f.Tag.Get("default")
+ }
+
+ t := reflect.TypeOf(sc).Elem()
+ p := reflect.ValueOf(sc).Elem()
+ for i := 0; i < p.NumField(); i++ {
+ field := t.Field(i)
+ value := reflect.ValueOf(valueFromStructTag(field))
+ p.Field(i).Set(value)
+ }
+
+ return sc
+}
+
+// AddFlags adds SourceConfig flags to the given FlagSet.
+func (src *SourceConfig) AddFlags(_ *flag.FlagSet) {
+ flag.StringVar(&src.Scheme, "src_scheme", src.Scheme, "source scheme\nenv: NVDSYNC_SCHEME")
+ flag.StringVar(&src.Host, "src_host", src.Host, "source host\nenv: NVDSYNC_HOST")
+ flag.StringVar(&src.CVEFeedPath, "src_cve_feed_path", src.CVEFeedPath, "source path for CVE feeds\nenv: NVDSYNC_CVE_FEED_PATH")
+ flag.StringVar(&src.CPEFeedPath, "src_cpe_feed_path", src.CPEFeedPath, "source path for CPE feeds\nenv: NVDSYNC_CPE_FEED_PATH")
+}
diff --git a/server/vulnerabilities/nvd/tools/providers/nvd/sync.go b/server/vulnerabilities/nvd/tools/providers/nvd/sync.go
new file mode 100644
index 0000000000..e251a5686f
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/providers/nvd/sync.go
@@ -0,0 +1,72 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nvd
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "strings"
+)
+
+// Syncer is an abstract interface for data feed synchronizers.
+type Syncer interface {
+ Sync(ctx context.Context, src SourceConfig, localdir string) error
+}
+
+// SyncError accumulates errors occured during Sync.Do() call.
+type SyncError []string
+
+// Error implements error interface.
+func (se SyncError) Error() string {
+ if len(se) == 0 {
+ return ""
+ }
+ sfx := ""
+ if len(se) > 1 {
+ sfx = "s"
+ }
+ return fmt.Sprintf("%d synchronisation error%s:\n\t%s", len(se), sfx, strings.Join(se, "\n\t"))
+}
+
+// Sync provides full synchronization between remote and local data feeds.
+type Sync struct {
+ Feeds []Syncer
+ Source *SourceConfig
+ LocalDir string
+}
+
+// Do executes the synchronization.
+func (s Sync) Do(ctx context.Context) error {
+ err := os.MkdirAll(s.LocalDir, 0755)
+ if err != nil {
+ return err
+ }
+ src := s.Source
+ if src == nil {
+ src = NewSourceConfig()
+ }
+ vsrc := *src
+ var errors SyncError
+ for _, feed := range s.Feeds {
+ if err = feed.Sync(ctx, vsrc, s.LocalDir); err != nil {
+ errors = append(errors, err.Error())
+ }
+ }
+ if len(errors) == 0 {
+ return nil
+ }
+ return errors
+}
diff --git a/server/vulnerabilities/nvd/tools/providers/nvd/xrename.go b/server/vulnerabilities/nvd/tools/providers/nvd/xrename.go
new file mode 100644
index 0000000000..36aa674c4c
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/providers/nvd/xrename.go
@@ -0,0 +1,51 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nvd
+
+import (
+ "fmt"
+ "io"
+ "os"
+)
+
+// xRename tries to rename oldpath to newpath, if it gets LinkError (most often
+// because of the files located on a different device) it copies and removes
+// it instead
+func xRename(oldpath, newpath string) error {
+ err := os.Rename(oldpath, newpath)
+ if _, ok := err.(*os.LinkError); ok {
+ var oldfile, newfile *os.File
+ if oldfile, err = os.Open(oldpath); err != nil {
+ return err
+ }
+ defer oldfile.Close()
+ var finfo os.FileInfo
+ if finfo, err = oldfile.Stat(); err != nil {
+ return err
+ }
+ if !finfo.Mode().IsRegular() {
+ return fmt.Errorf("failed to rename %q to %q: source file is not a regular file", oldpath, newpath)
+ }
+ if newfile, err = os.OpenFile(newpath, os.O_WRONLY|os.O_CREATE, finfo.Mode().Perm()); err != nil {
+ return err
+ }
+ defer newfile.Close()
+ if _, err = io.Copy(newfile, oldfile); err != nil {
+ return err
+ }
+ err = os.Remove(oldpath)
+ }
+ return err
+}
diff --git a/server/vulnerabilities/nvd/tools/wfn/doc.go b/server/vulnerabilities/nvd/tools/wfn/doc.go
new file mode 100644
index 0000000000..7fbea8d7be
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/wfn/doc.go
@@ -0,0 +1,18 @@
+// Package wfn provides a representation, bindings and matching of the Well-Formed CPE names as per
+// https://nvlpubs.nist.gov/nistpubs/Legacy/IR/nistir7695.pdf and
+// https://nvlpubs.nist.gov/nistpubs/Legacy/IR/nistir7696.pdf
+//
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package wfn
diff --git a/server/vulnerabilities/nvd/tools/wfn/fsb.go b/server/vulnerabilities/nvd/tools/wfn/fsb.go
new file mode 100644
index 0000000000..7bac5626ae
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/wfn/fsb.go
@@ -0,0 +1,166 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package wfn
+
+import (
+ "fmt"
+ "strings"
+ "unicode"
+)
+
+// BindToFmtString binds WFN to formatted string
+func (a Attributes) BindToFmtString() string {
+ parts := make([]string, 11)
+ for i, s := range []string{
+ a.Part,
+ a.Vendor,
+ a.Product,
+ a.Version,
+ a.Update,
+ a.Edition,
+ a.Language,
+ a.SWEdition,
+ a.TargetSW,
+ a.TargetHW,
+ a.Other,
+ } {
+ parts[i] = bindValueFS(s)
+ }
+ return fsbPrefix + strings.Join(parts, ":")
+}
+
+// UnbindFmtString loads WFN from formatted string
+func UnbindFmtString(s string) (*Attributes, error) {
+ if !strings.HasPrefix(s, fsbPrefix) {
+ return nil, fmt.Errorf("bad prefix in FSB %q", s)
+ }
+ attr := &Attributes{}
+ for i, partN := len(fsbPrefix), 0; i < len(s); i, partN = i+1, partN+1 {
+ var err error
+ switch partN {
+ case 0:
+ attr.Part, i, err = unbindValueFSAt(s, i)
+ case 1:
+ attr.Vendor, i, err = unbindValueFSAt(s, i)
+ case 2:
+ attr.Product, i, err = unbindValueFSAt(s, i)
+ case 3:
+ attr.Version, i, err = unbindValueFSAt(s, i)
+ case 4:
+ attr.Update, i, err = unbindValueFSAt(s, i)
+ case 5:
+ attr.Edition, i, err = unbindValueFSAt(s, i)
+ case 6:
+ attr.Language, i, err = unbindValueFSAt(s, i)
+ case 7:
+ attr.SWEdition, i, err = unbindValueFSAt(s, i)
+ case 8:
+ attr.TargetSW, i, err = unbindValueFSAt(s, i)
+ case 9:
+ attr.TargetHW, i, err = unbindValueFSAt(s, i)
+ case 10:
+ attr.Other, i, err = unbindValueFSAt(s, i)
+ }
+ if err != nil {
+ return nil, fmt.Errorf("unbind formatted string: %v", err)
+ }
+ }
+ return attr, nil
+}
+
+// StripSlashes removes escaping of punctuation characters from attribute value
+func StripSlashes(s string) string {
+ out := make([]byte, 0, len(s)) // might be more than we need, but no reallocs
+ for i := 0; i < len(s); i++ {
+ if s[i] == '\\' && i < len(s)-1 {
+ switch s[i+1] {
+ case '.', '_', '-': // these pass unquoted
+ continue
+ }
+ }
+ out = append(out, s[i])
+ }
+ return string(out)
+}
+
+func bindValueFS(s string) string {
+ switch s {
+ case Any:
+ return "*"
+ case NA:
+ return "-"
+ default:
+ return StripSlashes(s)
+ }
+}
+
+func unbindValueFSAt(s string, at int) (string, int, error) {
+ if len(s)-at < 1 || s[at] == ':' {
+ return Any, at, fmt.Errorf("could not unbind attribute at pos %d", at)
+ }
+ if len(s)-at == 1 || s[at+1] == ':' {
+ switch s[at] {
+ case '*':
+ return Any, at + 1, nil
+ case '-':
+ return NA, at + 1, nil
+ default:
+ return s[at : at+1], at + 1, nil
+ }
+ }
+ return addSlashesAt(s, at)
+}
+
+func addSlashesAt(s string, at int) (string, int, error) {
+ b := make([]byte, 0, len(s)*2) // assume a quote for every character
+ embedded := false
+ i := at
+ for ; i < len(s) && s[i] != ':'; i++ {
+ c := s[i]
+ if unicode.IsLetter(rune(c)) || unicode.IsDigit(rune(c)) || c == '_' {
+ b = append(b, c)
+ embedded = true
+ continue
+ }
+ switch c {
+ case '\\':
+ i++
+ if i == len(s) {
+ return "", i, fmt.Errorf("unquoted '\\' at the end of the FSB fragment: %q", s)
+ }
+ b = append(b, c, s[i])
+ embedded = true
+ case '*':
+ // An unquoted asterisk must appear at the beginning or end of the string
+ if i != at && i != len(s)-1 && s[i+1] != ':' {
+ return Any, i, fmt.Errorf("unquoted '*' inside the FSB fragment: %q", s)
+ }
+ b = append(b, c)
+ embedded = true
+ case '?':
+ if !(i == at || i == len(s)-1 || s[i+1] == ':' || // at the beginning or at the end of the string
+ (!embedded && i > 0 && s[i-1] == c || // not embedded and preceded by the same symbol
+ (embedded && s[i+1] == c))) { // embedded and followed by the same symbol
+ return Any, i, fmt.Errorf("unquoted '?' inside the FSB fragment %q (%t, %d)", s, embedded, i)
+ }
+ b = append(b, c)
+ embedded = false
+ default:
+ b = append(b, '\\', c)
+ embedded = true
+ }
+ }
+ return string(append([]byte{}, b...)), i, nil
+}
diff --git a/server/vulnerabilities/nvd/tools/wfn/fsb_test.go b/server/vulnerabilities/nvd/tools/wfn/fsb_test.go
new file mode 100644
index 0000000000..235ae7f180
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/wfn/fsb_test.go
@@ -0,0 +1,106 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package wfn
+
+import (
+ "fmt"
+ "testing"
+)
+
+func TestUnbindFmtString(t *testing.T) {
+ cases := []struct {
+ FSB string
+ Expect string
+ Fail bool
+ }{
+ {
+ FSB: "cpe:2.3:a:microsoft:internet_explorer:8.0.6001:beta:*:*:*:*:*:*",
+ Expect: `wfn:[part="a",vendor="microsoft",product="internet_explorer",version="8\.0\.6001",update="beta",edition=ANY,language=ANY]`,
+ },
+ {
+ FSB: "cpe:2.3:a:microsoft:internet_exp?????:8.*:sp?:*:*:*:*:*:*",
+ Expect: `wfn:[part="a",vendor="microsoft",product="internet_exp?????",version="8\.*",update="sp?",edition=ANY,language=ANY]`,
+ },
+ {
+ FSB: "cpe:2.3:a:microsoft:internet_explorer:8.*:sp?:*:*:*:*:*:*",
+ Expect: `wfn:[part="a",vendor="microsoft",product="internet_explorer",version="8\.*",update="sp?",edition=ANY,language=ANY]`,
+ },
+ {
+ FSB: "cpe:2.3:a:hp:insight_diagnostics:7.4.0.1570:-:*:*:online:win2003:x64:*",
+ Expect: `wfn:[part="a",vendor="hp",product="insight_diagnostics",version="7\.4\.0\.1570",update=NA,edition=ANY,sw_edition="online",target_sw="win2003",target_hw="x64",other=ANY,language=ANY]`,
+ },
+ {
+ FSB: `cpe:2.3:a:foo\\bar:big\$money:2010:*:*:*:special:ipod_touch:80gb:*`,
+ Expect: `wfn:[part="a",vendor="foo\\bar",product="big\$money",version="2010",update=ANY,edition=ANY,sw_edition="special",target_sw="ipod_touch",target_hw="80gb",other=ANY,language=ANY]`,
+ },
+ {
+ FSB: `cpe:2.3:a:cisco:cisco_security_monitoring\`,
+ Fail: true,
+ },
+ {
+ FSB: `cpe:2.3:a:disney:where\\'s_my_perry?_free:1.5.1:*:*:*:*:android:*:*`,
+ Fail: true,
+ },
+ {
+ FSB: "cpe:2.3:a:hp:insight_diagnostics:7.4.*.1570:*:*:*:*:*:*",
+ Fail: true,
+ },
+ }
+ for _, tc := range cases {
+ tc := tc
+ t.Run(tc.FSB, func(t *testing.T) {
+ attr, err := UnbindFmtString(tc.FSB)
+ if err != nil {
+ if tc.Fail {
+ return
+ }
+ t.Fatalf("failed to parse FSB %q: %v", tc.FSB, err)
+ }
+ if tc.Fail {
+ t.Fatalf("FSB parsed successfully, despite failure was expected: %q", tc.FSB)
+ }
+ if attr.String() != tc.Expect {
+ t.Fatalf("expected %s\ngot %s", tc.Expect, attr)
+ }
+ })
+ }
+}
+
+func BenchmarkUnbindFmtString(t *testing.B) {
+ for i := 0; i < t.N; i++ {
+ _, _ = UnbindFmtString("cpe:2.3:a:hp:insight_diagnostics:7.4.0.1570:-:*:*:online:win2003:x64:*")
+ }
+}
+
+func TestBindToFmtString(t *testing.T) {
+ cases := []string{
+ "cpe:2.3:a:microsoft:internet_explorer:8.0.6001:beta:*:*:*:*:*:*",
+ "cpe:2.3:a:microsoft:internet_explorer:8.*:sp?:*:*:*:*:*:*",
+ "cpe:2.3:a:hp:insight_diagnostics:7.4.0.1570:-:*:*:online:win2003:x64:*",
+ `cpe:2.3:a:foo\\bar:big\$\*\?money:2010:*:*:*:special:ipod_touch:80gb:*`,
+ }
+ for n, c := range cases {
+ c := c
+ t.Run(fmt.Sprintf("case#%d", n), func(t *testing.T) {
+ attr, err := UnbindFmtString(c)
+ if err != nil {
+ t.Fatalf("failed to parse test input %q: %v", c, err)
+ }
+ if out := attr.BindToFmtString(); out != c {
+ t.Fatalf("expected %s\ngot %s", c, out)
+ }
+ })
+ }
+}
diff --git a/server/vulnerabilities/nvd/tools/wfn/matcher.go b/server/vulnerabilities/nvd/tools/wfn/matcher.go
new file mode 100644
index 0000000000..fca69e6e47
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/wfn/matcher.go
@@ -0,0 +1,120 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package wfn
+
+// Matcher knows whether it matches some attributes
+type Matcher interface {
+ // Match returns attributes which match it
+ // if require version, then Matcher which matches all versions should return false
+ Match(attrs []*Attributes, requireVersion bool) (matches []*Attributes)
+ // Config returns all attributes that are used by in the matching process
+ Config() []*Attributes
+}
+
+// Attrs is part of the Matcher interface
+func (a *Attributes) Config() []*Attributes {
+ return []*Attributes{a}
+}
+
+// MatchOnlyVersion checks whether version matches
+func (a *Attributes) MatchOnlyVersion(attr *Attributes) bool {
+ if a == nil || attr == nil {
+ return a == attr // both are nil
+ }
+ return matchAttr(a.Version, attr.Version)
+}
+
+// MatchWithoutVersion checks whether everything else besides the version matches
+func (a *Attributes) MatchWithoutVersion(attr *Attributes) bool {
+ if a == nil || attr == nil {
+ return a == attr // both are nil
+ }
+ return matchAttr(a.Product, attr.Product) &&
+ matchAttr(a.Vendor, attr.Vendor) && matchAttr(a.Part, attr.Part) &&
+ matchAttr(a.Update, attr.Update) && matchAttr(a.Edition, attr.Edition) &&
+ matchAttr(a.Language, attr.Language) && matchAttr(a.SWEdition, attr.SWEdition) &&
+ matchAttr(a.TargetHW, attr.TargetHW) && matchAttr(a.TargetSW, attr.TargetSW) &&
+ matchAttr(a.Other, attr.Other)
+}
+
+// MatchAll returns a Matcher which matches only if all matchers match
+func MatchAll(ms ...Matcher) Matcher {
+ return &multiMatcher{ms, true}
+}
+
+// MatchAll returns a Matcher which matches if any of the matchers match
+func MatchAny(ms ...Matcher) Matcher {
+ return &multiMatcher{ms, false}
+}
+
+// DontMatch returns a Matcher which matches if the given matchers doesn't
+func DontMatch(m Matcher) Matcher {
+ return notMatcher{m}
+}
+
+type multiMatcher struct {
+ matchers []Matcher
+ // if true, match will only return something if all matchers matched at least something
+ allMatch bool
+}
+
+// Match is part of the Matcher interface
+func (mm *multiMatcher) Match(attrs []*Attributes, requireVersion bool) []*Attributes {
+ matched := make(map[*Attributes]bool)
+ for _, matcher := range mm.matchers {
+ matches := matcher.Match(attrs, requireVersion)
+ if mm.allMatch && len(matches) == 0 {
+ // all matchers need to match at least one attr
+ return nil
+ }
+ for _, m := range matches {
+ matched[m] = true
+ }
+ }
+
+ matches := make([]*Attributes, 0, len(matched))
+ for m := range matched {
+ matches = append(matches, m)
+ }
+ return matches
+}
+
+// Attrs is part of the Matcher interface
+func (mm *multiMatcher) Config() []*Attributes {
+ var attrs []*Attributes
+ for _, matcher := range mm.matchers {
+ attrs = append(attrs, matcher.Config()...)
+ }
+ return attrs
+}
+
+type notMatcher struct {
+ Matcher
+}
+
+// Match is part of the Matcher interface
+func (nm notMatcher) Match(attrs []*Attributes, requireVersion bool) (matches []*Attributes) {
+ matched := make(map[*Attributes]bool)
+ for _, m := range nm.Matcher.Match(attrs, requireVersion) {
+ matched[m] = true
+ }
+
+ for _, a := range attrs {
+ if !matched[a] {
+ matches = append(matches, a)
+ }
+ }
+ return matches
+}
diff --git a/server/vulnerabilities/nvd/tools/wfn/matching.go b/server/vulnerabilities/nvd/tools/wfn/matching.go
new file mode 100644
index 0000000000..cdb999fb40
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/wfn/matching.go
@@ -0,0 +1,391 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package wfn
+
+import (
+ "errors"
+ "fmt"
+)
+
+// Possible values of Relation type
+const (
+ Disjoint Relation = iota
+ Subset
+ Equal
+ Superset
+)
+
+// HasWildcard returns true if attribute has a wildcard symbol in it
+func HasWildcard(s string) bool {
+ for n, r := range s {
+ if r != '*' && r != '?' {
+ continue
+ }
+ quoted := false
+ for i := n - 1; i >= 0; i-- {
+ if s[i] != '\\' {
+ break
+ }
+ quoted = !quoted
+ }
+ if !quoted {
+ return true
+ }
+ }
+ return false
+}
+
+// Relation describes four possible set relations of wfns attribute-value
+type Relation int
+
+// String return human readable representation of Relation value
+func (r Relation) String() string {
+ switch r {
+ case Disjoint:
+ return "DISJOINT"
+ case Subset:
+ return "SUBSET"
+ case Equal:
+ return "EQUAL"
+ case Superset:
+ return "SUPERSET"
+ default:
+ return fmt.Sprintf("Undefined value %d", r)
+ }
+}
+
+// Comparison is the result of CPE name matching
+type Comparison struct {
+ Part Relation
+ Vendor Relation
+ Product Relation
+ Version Relation
+ Update Relation
+ Edition Relation
+ Language Relation
+ SWEdition Relation
+ TargetSW Relation
+ TargetHW Relation
+ Other Relation
+}
+
+// IsDisjoint returns true if the result CPE name matching is disjoint
+func (c Comparison) IsDisjoint() bool {
+ switch {
+ case c.Part == Disjoint:
+ return true
+ case c.Vendor == Disjoint:
+ return true
+ case c.Product == Disjoint:
+ return true
+ case c.Version == Disjoint:
+ return true
+ case c.Update == Disjoint:
+ return true
+ case c.Edition == Disjoint:
+ return true
+ case c.Language == Disjoint:
+ return true
+ case c.SWEdition == Disjoint:
+ return true
+ case c.TargetSW == Disjoint:
+ return true
+ case c.TargetHW == Disjoint:
+ return true
+ case c.Other == Disjoint:
+ return true
+ default:
+ return false
+ }
+}
+
+// IsEqual returns true if the result CPE name matching is equal
+func (c Comparison) IsEqual() bool {
+ switch {
+ case c.Part != Equal:
+ return false
+ case c.Vendor != Equal:
+ return false
+ case c.Product != Equal:
+ return false
+ case c.Version != Equal:
+ return false
+ case c.Update != Equal:
+ return false
+ case c.Edition != Equal:
+ return false
+ case c.Language != Equal:
+ return false
+ case c.SWEdition != Equal:
+ return false
+ case c.TargetSW != Equal:
+ return false
+ case c.TargetHW != Equal:
+ return false
+ case c.Other != Equal:
+ return false
+ default:
+ return true
+ }
+}
+
+// IsSubset returns true if the result CPE name matching is a subset relation
+func (c Comparison) IsSubset() bool {
+ switch {
+ case c.Part != Equal && c.Part != Subset:
+ return false
+ case c.Vendor != Equal && c.Vendor != Subset:
+ return false
+ case c.Product != Equal && c.Product != Subset:
+ return false
+ case c.Version != Equal && c.Version != Subset:
+ return false
+ case c.Update != Equal && c.Update != Subset:
+ return false
+ case c.Edition != Equal && c.Edition != Subset:
+ return false
+ case c.Language != Equal && c.Language != Subset:
+ return false
+ case c.SWEdition != Equal && c.SWEdition != Subset:
+ return false
+ case c.TargetSW != Equal && c.TargetSW != Subset:
+ return false
+ case c.TargetHW != Equal && c.TargetHW != Subset:
+ return false
+ case c.Other != Equal && c.Other != Subset:
+ return false
+ default:
+ return true
+ }
+}
+
+// IsSuperset returns true if the result CPE name matching is a superset relation
+func (c Comparison) IsSuperset() bool {
+ switch {
+ case c.Part != Equal && c.Part != Superset:
+ return false
+ case c.Vendor != Equal && c.Vendor != Superset:
+ return false
+ case c.Product != Equal && c.Product != Superset:
+ return false
+ case c.Version != Equal && c.Version != Superset:
+ return false
+ case c.Update != Equal && c.Update != Superset:
+ return false
+ case c.Edition != Equal && c.Edition != Superset:
+ return false
+ case c.Language != Equal && c.Language != Superset:
+ return false
+ case c.SWEdition != Equal && c.SWEdition != Superset:
+ return false
+ case c.TargetSW != Equal && c.TargetSW != Superset:
+ return false
+ case c.TargetHW != Equal && c.TargetHW != Superset:
+ return false
+ case c.Other != Equal && c.Other != Superset:
+ return false
+ default:
+ return true
+ }
+}
+
+// Relation returns relation between matched CPE names
+func (c Comparison) Relation() Relation {
+ if c.IsSubset() {
+ return Subset
+ }
+ if c.IsEqual() {
+ return Equal
+ }
+ if c.IsSuperset() {
+ return Superset
+ }
+ return Disjoint
+}
+
+// Compare performs comparison of each attribute-value (A-V) of the wfns
+// as per Name Matching Specification v.2.3 and returns the set relation between
+// source and target attribute-values.
+// The table below illustrates a set of source and target A-Vs and the resulting set of attribute
+// comparison relations.
+// +--------------------------------------------+------------------------------------+
+// | Attribute Relation Set | Name Comparison Relation |
+// +--------------------------------------------+------------------------------------+
+// | any attribute relation is != | CPE name relation is DISJOINT (!=) |
+// | all attribute relations are == | CPE name relation is EQAL (==) |
+// | all attribute relations are Subset or == | CPE name relation is Subset |
+// | all attribute relations are Superset or == | CPE name relation is Superset |
+// +--------------------------------------------+------------------------------------+
+func Compare(src, tgt *Attributes) (Comparison, error) {
+ var result Comparison
+ var err error
+ if result.Part, err = CompareAttr(src.Part, tgt.Part); err != nil {
+ return result, fmt.Errorf("failed to compare wfns %q to %q: %v", src.Part, tgt.Part, err)
+ }
+ if result.Vendor, err = CompareAttr(src.Vendor, tgt.Vendor); err != nil {
+ return result, fmt.Errorf("failed to compare wfns %q to %q: %v", src.Vendor, tgt.Vendor, err)
+ }
+ if result.Product, err = CompareAttr(src.Product, tgt.Product); err != nil {
+ return result, fmt.Errorf("failed to compare wfns %q to %q: %v", src.Product, tgt.Product, err)
+ }
+ if result.Version, err = CompareAttr(src.Version, tgt.Version); err != nil {
+ return result, fmt.Errorf("failed to compare wfns %q to %q: %v", src.Version, tgt.Version, err)
+ }
+ if result.Update, err = CompareAttr(src.Update, tgt.Update); err != nil {
+ return result, fmt.Errorf("failed to compare wfns %q to %q: %v", src.Update, tgt.Update, err)
+ }
+ if result.Edition, err = CompareAttr(src.Edition, tgt.Edition); err != nil {
+ return result, fmt.Errorf("failed to compare wfns %q to %q: %v", src.Edition, tgt.Edition, err)
+ }
+ if result.Language, err = CompareAttr(src.Language, tgt.Language); err != nil {
+ return result, fmt.Errorf("failed to compare wfns %q to %q: %v", src.Language, tgt.Language, err)
+ }
+ if result.SWEdition, err = CompareAttr(src.SWEdition, tgt.SWEdition); err != nil {
+ return result, fmt.Errorf("failed to compare wfns %q to %q: %v", src.SWEdition, tgt.SWEdition, err)
+ }
+ if result.TargetSW, err = CompareAttr(src.TargetSW, tgt.TargetSW); err != nil {
+ return result, fmt.Errorf("failed to compare wfns %q to %q: %v", src.TargetSW, tgt.TargetSW, err)
+ }
+ if result.TargetHW, err = CompareAttr(src.TargetHW, tgt.TargetHW); err != nil {
+ return result, fmt.Errorf("failed to compare wfns %q to %q: %v", src.TargetHW, tgt.TargetHW, err)
+ }
+ if result.Other, err = CompareAttr(src.Other, tgt.Other); err != nil {
+ return result, fmt.Errorf("failed to compare wfns %q to %q: %v", src.Other, tgt.Other, err)
+ }
+ return result, nil
+}
+
+// Match returns false if the src and tgt attributes are disjoint.
+// Undefined relations between attributes (see CompareAttr) are considered to be disjoint,
+// except when source attribute matches target attribute byte-by-byte.
+func Match(src, tgt *Attributes) bool {
+ if src == nil || tgt == nil {
+ return false
+ }
+ return matchAttr(src.Part, tgt.Part) && matchAttr(src.Vendor, tgt.Vendor) &&
+ matchAttr(src.Product, tgt.Product) && matchAttr(src.Version, tgt.Version) &&
+ matchAttr(src.Update, tgt.Update) && matchAttr(src.Edition, tgt.Edition) &&
+ matchAttr(src.Language, tgt.Language) && matchAttr(src.SWEdition, tgt.SWEdition) &&
+ matchAttr(src.TargetHW, tgt.TargetHW) && matchAttr(src.TargetSW, tgt.TargetSW) &&
+ matchAttr(src.Other, tgt.Other)
+}
+
+// CompareAttr calculates a relation between a pair of wfn attribute values.
+// Accordingly to standard, string matching must be insensitive to lexical case,
+// target A-V must not have wildcards.
+// The table below defines possible set relations for each comparison
+// ANY and NA are logical values as defined per [CPE23-N:5.3.1]
+// i and k are wildcard-free attribute-value strings that are not identical, e.g. i is "foo" and k is "bar"
+// m + w is attribute-value string containing a legal combination of unquoted question mark or asterisk wildcards
+//
+// at the beginning and/or the end of the string, e.g. "*b??"
+// Enumeration of
+// Attribute Comparison Set Relations
+//
+// +------------+------------+--------------+
+// | Source A-V | Target A-V | Relation |
+// +------------+------------+--------------+
+// | ANY | ANY | == |
+// | ANY | NA | Superset |
+// | ANY | i | Superset |
+// | ANY | m + w | undef |
+// | NA | ANY | Subset |
+// | NA | NA | == |
+// | NA | i | != |
+// | NA | m + w | undefined |
+// | i | i | == |
+// | i | k | != |
+// | i | m + w | undefined |
+// | i | NA | != |
+// | i | ANY | Subset |
+// | m1 + w | m2 | Subset or != |
+// | m + w | ANY | Subset |
+// | m1 + w | NA | != |
+// | m1 + w | m2 + w | undefined |
+// +----------------------------------------+
+func CompareAttr(src, tgt string) (Relation, error) {
+ if src != NA && src != Any && HasWildcard(tgt) {
+ return Disjoint, errors.New("target attribute value cannot contain wildcard")
+ }
+ if src == tgt {
+ return Equal, nil
+ }
+ if src == Any {
+ return Superset, nil
+ }
+ if tgt == Any {
+ return Subset, nil
+ }
+ if src == NA || tgt == NA {
+ return Disjoint, nil
+ }
+ return matchStr(src, tgt), nil
+}
+
+// matchAttr returns true if relation between src and tgt is one of Equal, Subset or Superset.
+// It returns false on undefined relations, except when src == tgt byte-by-byte.
+// This is crude but fast(-er) version of CompareAttr.
+func matchAttr(src, tgt string) bool {
+ switch {
+ case src == Any || tgt == Any || src == tgt:
+ return true
+ case src == NA || tgt == NA || HasWildcard(tgt):
+ return false
+ default:
+ return matchStr(src, tgt) != Disjoint
+ }
+}
+
+func matchStr(s, t string) Relation {
+ escaped := false
+ matchesAs := Equal
+ idx := 0
+ for ; idx < len(t); idx++ {
+ if idx >= len(s) {
+ return Disjoint
+ }
+ if !escaped && s[idx] == '*' {
+ if idx == len(s)-1 {
+ return Superset
+ }
+ for i := idx; i < len(t); i++ {
+ if matchStr(s[idx+1:], t[i:]) != Disjoint {
+ return Superset
+ }
+ }
+ return Disjoint
+ }
+
+ if (escaped || s[idx] != '?') && s[idx] != t[idx] {
+ return Disjoint
+ } else if !escaped && s[idx] == '?' {
+ matchesAs = Superset
+ }
+ if s[idx] == '\\' {
+ escaped = !escaped
+ } else {
+ escaped = false
+ }
+ }
+ for ; idx < len(s); idx++ {
+ if s[idx] != '*' {
+ return Disjoint
+ }
+ }
+ if len(s) > len(t) {
+ return Superset
+ }
+ return matchesAs
+}
diff --git a/server/vulnerabilities/nvd/tools/wfn/matching_test.go b/server/vulnerabilities/nvd/tools/wfn/matching_test.go
new file mode 100644
index 0000000000..63806e7b12
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/wfn/matching_test.go
@@ -0,0 +1,185 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package wfn
+
+import (
+ "fmt"
+ "testing"
+)
+
+func TestHasWildcard(t *testing.T) {
+ cases := []struct {
+ Src string
+ Expect bool
+ }{
+ {"", false},
+ {"foo", false},
+ {"bar*", true},
+ {"?baz", true},
+ {`\\\\*foo`, true},
+ {`bar\\\?`, false},
+ {`foo\bar*`, true},
+ {`b\?r?`, true},
+ }
+ for _, c := range cases {
+ t.Run(c.Src, func(t *testing.T) {
+ r := HasWildcard(c.Src)
+ if r != c.Expect {
+ t.Fatalf("HasWildcard(%q) returned %v, %v was expected", c.Src, r, c.Expect)
+ }
+ })
+ }
+}
+
+func TestMatchStr(t *testing.T) {
+ cases := []struct {
+ Src string
+ Tgt string
+ Expect Relation
+ }{
+ {"foo", "bar", Disjoint},
+ {"bar", "bar", Equal},
+ {"*", "foo", Superset},
+ {"*a?", "bar", Superset},
+ {"*", "", Superset},
+ {"f*", "foo", Superset},
+ {"ba?", "bar", Superset},
+ {"fo??", "foo", Disjoint},
+ {"foo*", "foo", Superset},
+ {"*bar", "bar", Superset},
+ {"??o", "foo", Superset},
+ {"??o", "bar", Disjoint},
+ {"boo\\?", "boo\\?", Equal},
+ }
+ for _, c := range cases {
+ t.Run(fmt.Sprintf("%q vs %q", c.Src, c.Tgt), func(t *testing.T) {
+ r := matchStr(c.Src, c.Tgt)
+ if r != c.Expect {
+ t.Fatalf("matchStr returned %v, %v was expected", r, c.Expect)
+ }
+ })
+ }
+}
+
+func TestCompare(t *testing.T) {
+ cases := []struct {
+ Src string
+ Tgt string
+ Fail bool
+ Expect Relation
+ }{
+ {
+ Src: `cpe:2.3:a:microsoft:internet_explorer:8.*:sp?:*:*:*:*:*:*`,
+ Tgt: `cpe:2.3:a:microsoft:internet_explorer:8.0.6001:sp3:*:*:*:*:*:*`,
+ Expect: Superset,
+ },
+ }
+ for _, c := range cases {
+ t.Run(fmt.Sprintf("%q vs %q", c.Src, c.Tgt), func(t *testing.T) {
+ srcAttr, err := UnbindFmtString(c.Src)
+ if err != nil {
+ t.Fatalf("failed to unbind WFN from FSB %q: %v", c.Src, err)
+ }
+ tgtAttr, err := UnbindFmtString(c.Tgt)
+ if err != nil {
+ t.Fatalf("failed to unbind WFN from FSB %q: %v", c.Tgt, err)
+ }
+ r, err := Compare(srcAttr, tgtAttr)
+ if c.Fail && err == nil {
+ t.Fatal("test was expected to fail, but succeeded")
+ }
+ if !c.Fail && err != nil {
+ t.Fatalf("test was expected to succeed, but failed: %v", err)
+ }
+ if r.Relation() != c.Expect {
+ t.Fatalf("Compare returned %v (%v), %v was expected", r.Relation(), r, c.Expect)
+ }
+ })
+ }
+}
+
+func BenchmarkCompare(b *testing.B) {
+ src := `cpe:2.3:a:microsoft:*internet_ex??????:8.0.*:sp?:*:*:*:*:*:*`
+ tgt := `cpe:2.3:a:microsoft:internet_explorer:8.1.6001:sp3:*:*:*:*:*:*`
+ srcAttr, err := UnbindFmtString(src)
+ if err != nil {
+ b.Fatalf("failed to unbind WFN from FSB %q: %v", src, err)
+ }
+ tgtAttr, err := UnbindFmtString(tgt)
+ if err != nil {
+ b.Fatalf("failed to unbind WFN from FSB %q: %v", tgt, err)
+ }
+ for i := 0; i < b.N; i++ {
+ // checking error and result adds about 10% of runtime to this benchmark on my machine
+ // and correctness is covered by tests, so skip it
+ _, _ = Compare(srcAttr, tgtAttr)
+ }
+}
+
+func BenchmarkMatch(b *testing.B) {
+ src := `cpe:2.3:a:microsoft:*internet_ex??????:8.0.*:sp?:*:*:*:*:*:*`
+ tgt := `cpe:2.3:a:microsoft:internet_explorer:8.1.6001:sp3:*:*:*:*:*:*`
+ srcAttr, err := UnbindFmtString(src)
+ if err != nil {
+ b.Fatalf("failed to unbind WFN from FSB %q: %v", src, err)
+ }
+ tgtAttr, err := UnbindFmtString(tgt)
+ if err != nil {
+ b.Fatalf("failed to unbind WFN from FSB %q: %v", tgt, err)
+ }
+ for i := 0; i < b.N; i++ {
+ // checking error and result adds about 10% of runtime to this benchmark on my machine
+ // and correctness is covered by tests, so skip it
+ Match(srcAttr, tgtAttr)
+ }
+}
+
+func BenchmarkIsDisjoint(b *testing.B) {
+ src := `cpe:2.3:a:microsoft:*internet_ex??????:8.*:sp?:*:*:*:*:*:1`
+ tgt := `cpe:2.3:a:microsoft:internet_explorer:8.0.6001:sp3:*:*:*:*:*:2`
+ srcAttr, err := UnbindFmtString(src)
+ if err != nil {
+ b.Fatalf("failed to unbind WFN from FSB %q: %v", src, err)
+ }
+ tgtAttr, err := UnbindFmtString(tgt)
+ if err != nil {
+ b.Fatalf("failed to unbind WFN from FSB %q: %v", tgt, err)
+ }
+ cmp, _ := Compare(srcAttr, tgtAttr)
+ for i := 0; i < b.N; i++ {
+ cmp.IsDisjoint()
+ }
+}
+
+func BenchmarkHasWildcard(b *testing.B) {
+ tests := map[string]string{
+ "has": `cpe:2.3:a:microsoft:*internet_ex??????:8.*:sp?:*:*:*:*:*:*`,
+ "has not": `cpe:2.3:a:microsoft:internet_explorer:8.0:sp2:*:*:*:*:*:*`,
+ "has escaped": `cpe:2.3:a:vendor\?:product\?:8.0:sp2:*:*:*:*:*:*`,
+ }
+ for tag, test := range tests {
+ b.Run(tag, func(b *testing.B) {
+ srcAttr, err := UnbindFmtString(test)
+ if err != nil {
+ b.Fatalf("failed to unbind WFN from FSB %q: %v", test, err)
+ }
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ HasWildcard(srcAttr.Vendor)
+ HasWildcard(srcAttr.Product)
+ }
+ })
+ }
+}
diff --git a/server/vulnerabilities/nvd/tools/wfn/uri.go b/server/vulnerabilities/nvd/tools/wfn/uri.go
new file mode 100644
index 0000000000..5e2f502f5f
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/wfn/uri.go
@@ -0,0 +1,346 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package wfn
+
+import (
+ "fmt"
+ "strconv"
+ "strings"
+)
+
+// BindToURI binds WFN to URI
+func (a Attributes) BindToURI() string {
+ var parts []string
+ for i, v := range []string{
+ a.Part,
+ a.Vendor,
+ a.Product,
+ a.Version,
+ a.Update,
+ "",
+ a.Language,
+ } {
+ if i != 5 { // other than edition
+ parts = append(parts, bindValueURI(v))
+ continue
+ }
+ edParts := make([]string, 5)
+ allNAs := true
+ for i, v2 := range []string{a.Edition, a.SWEdition, a.TargetSW, a.TargetHW, a.Other} {
+ edParts[i] = bindValueURI(v2)
+ if edParts[i] != "-" {
+ allNAs = false
+ }
+ }
+ if allNAs {
+ parts = append(parts, "-")
+ } else {
+ parts = append(parts, pack(edParts))
+ }
+ }
+ // empty elements at the end of the URI should be omitted
+ for i := len(parts) - 1; i >= 0; i-- {
+ if parts[i] != "" {
+ break
+ }
+ parts = parts[:i]
+ }
+ return uriPrefix + strings.Join(parts, ":")
+}
+
+// UnbindURI loads WFN from URI
+func UnbindURI(s string) (*Attributes, error) {
+ if !strings.HasPrefix(s, uriPrefix) {
+ return nil, fmt.Errorf("unbind uri: bad prefix in URI %q", s)
+ }
+ s = strings.ToLower(s[len(uriPrefix):]) // reject schema prefix + normalize
+ attr := Attributes{}
+ var err error
+ for i, partN := 0, 0; i < len(s); i, partN = i+1, partN+1 {
+ switch partN {
+ case 0:
+ attr.Part, i, err = unbindValueURIAtTill(s, i, ':')
+ case 1:
+ attr.Vendor, i, err = unbindValueURIAtTill(s, i, ':')
+ case 2:
+ attr.Product, i, err = unbindValueURIAtTill(s, i, ':')
+ case 3:
+ attr.Version, i, err = unbindValueURIAtTill(s, i, ':')
+ case 4:
+ attr.Update, i, err = unbindValueURIAtTill(s, i, ':')
+ case 5:
+ if s[i] != '~' {
+ attr.Edition, i, err = unbindValueURIAtTill(s, i, ':')
+ break
+ }
+ i++
+ edition23:
+ for subpartN := 0; i < len(s); i, subpartN = i+1, subpartN+1 {
+ switch subpartN {
+ case 0:
+ attr.Edition, i, err = unbindValueURIAtTill(s, i, '~')
+ case 1:
+ attr.SWEdition, i, err = unbindValueURIAtTill(s, i, '~')
+ case 2:
+ attr.TargetSW, i, err = unbindValueURIAtTill(s, i, '~')
+ case 3:
+ attr.TargetHW, i, err = unbindValueURIAtTill(s, i, '~')
+ case 4:
+ attr.Other, i, err = unbindValueURIAtTill(s, i, ':')
+ default:
+ break edition23
+ }
+ }
+ case 6:
+ attr.Language, i, err = unbindValueURIAtTill(s, i, ':')
+ }
+ if err != nil {
+ return nil, fmt.Errorf("unbind uri: %v", err)
+ }
+ }
+ return &attr, nil
+}
+
+func pack(ss []string) string {
+ compat := true
+ for _, s := range ss[1:] {
+ if s != "" {
+ compat = false
+ break
+ }
+ }
+ if compat {
+ return ss[0]
+ }
+ return "~" + strings.Join(ss, "~")
+}
+
+// Scans an input string s and applies the following transformations:
+// - pass alphanumeric characters thru untouched
+// - percent-encode quoted non-alphanumerics as needed
+// - unquoted special characters are mapped to their special forms.
+func bindValueURI(s string) string {
+ var out []byte
+ switch s {
+ case NA:
+ return "-"
+ case Any:
+ return ""
+ }
+ for i := 0; i < len(s); i++ {
+ b := s[i]
+ if b >= '0' && b <= '9' || b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z' || b == '_' {
+ // alnum + '_' pass untouched
+ out = append(out, b)
+ } else if b == '\\' {
+ // percent-encode escaped characters
+ // sanity check should be done during unbinding, so here we silently skip all
+ // illegal characters
+ i++
+ if i == len(s) {
+ break
+ }
+ out = append(out, pctEncode(s[i])...)
+ } else if b == '?' { // unquoted '?' -> "%01"
+ out = append(out, '%', '0', '1')
+ } else if b == '*' { // unquoted '*' -> "%02"
+ out = append(out, '%', '0', '2')
+ }
+ }
+ return string(out)
+}
+
+func unbindValueURIAtTill(s string, at int, till byte) (string, int, error) {
+ if at >= len(s) || s[at] == till {
+ return Any, at, nil
+ }
+ if s[at] == '-' {
+ return NA, at + 1, nil
+ }
+ out := make([]byte, 0, len(s)*2) // assume the worst
+ embedded := false
+ i := at
+loop:
+ for ; i < len(s); i++ {
+ switch s[i] {
+ case till:
+ break loop
+ case '%':
+ if i+3 > len(s) {
+ return "", i, fmt.Errorf("unbind URI attribute: illegal percent-encoded value at %d: %q", i, s[i:])
+ }
+ codeStr := s[i : i+3]
+ code, err := strconv.ParseInt(s[i+1:i+3], 16, 8)
+ if err != nil {
+ return "", i, fmt.Errorf("unbind URI attribute: illegal percent-encoded value at %d: %q", i, s[i+1:i+3])
+ }
+ if code == 0x1 || code == 0x2 {
+ if !(i == at || i == len(s)-3 || s[i+3] == till || // at the beginning or at the end of the string
+ (!embedded && i > 2 && s[i-3:i] == codeStr || // not embedded and preceded by the same symbol
+ (embedded && i+6 < len(s) && s[i+3:i+6] == codeStr))) { // embedded and followed by the same symbol
+ return "", i, fmt.Errorf("unbind URI attribute: %%%02d is embedded into string %q", code, s)
+ }
+ switch code {
+ case 0x1:
+ out = append(out, '?')
+ case 0x2:
+ out = append(out, '*')
+ }
+ i += 2
+ break
+ }
+ switch code {
+ case 0x21:
+ out = append(out, '\\', '!')
+ case 0x22:
+ out = append(out, '\\', '"')
+ case 0x23:
+ out = append(out, '\\', '#')
+ case 0x24:
+ out = append(out, '\\', '$')
+ case 0x25:
+ out = append(out, '\\', '%')
+ case 0x26:
+ out = append(out, '\\', '&')
+ case 0x27:
+ out = append(out, '\\', '\'')
+ case 0x28:
+ out = append(out, '\\', '(')
+ case 0x29:
+ out = append(out, '\\', ')')
+ case 0x2a:
+ out = append(out, '\\', '*')
+ case 0x2b:
+ out = append(out, '\\', '+')
+ case 0x2c:
+ out = append(out, '\\', ',')
+ case 0x2f:
+ out = append(out, '\\', '/')
+ case 0x3a:
+ out = append(out, '\\', ':')
+ case 0x3b:
+ out = append(out, '\\', ';')
+ case 0x3c:
+ out = append(out, '\\', '<')
+ case 0x3d:
+ out = append(out, '\\', '=')
+ case 0x3e:
+ out = append(out, '\\', '>')
+ case 0x3f:
+ out = append(out, '\\', '?')
+ case 0x40:
+ out = append(out, '\\', '@')
+ case 0x5b:
+ out = append(out, '\\', '[')
+ case 0x5c:
+ out = append(out, '\\', '\\')
+ case 0x5d:
+ out = append(out, '\\', ']')
+ case 0x5e:
+ out = append(out, '\\', '^')
+ case 0x60:
+ out = append(out, '\\', '`')
+ case 0x7b:
+ out = append(out, '\\', '{')
+ case 0x7c:
+ out = append(out, '\\', '|')
+ case 0x7d:
+ out = append(out, '\\', '}')
+ case 0x7e:
+ out = append(out, '\\', '~')
+ default:
+ return "", i, fmt.Errorf("unbind URI attribute: illegal percent-encoded value %q", s[i+1:i+3])
+ }
+ i += 2
+ embedded = true
+ case '.', '-', '~':
+ out = append(out, '\\', s[i])
+ embedded = true
+ default:
+ out = append(out, s[i])
+ embedded = true
+ }
+ }
+ return string(out), i, nil
+}
+
+func pctEncode(b byte) []byte {
+ switch b {
+ case '!':
+ return []byte("%21")
+ case '"':
+ return []byte("%22")
+ case '#':
+ return []byte("%23")
+ case '$':
+ return []byte("%24")
+ case '%':
+ return []byte("%25")
+ case '&':
+ return []byte("%26")
+ case '\'':
+ return []byte("%27")
+ case '(':
+ return []byte("%28")
+ case ')':
+ return []byte("%29")
+ case '*':
+ return []byte("%2a")
+ case '+':
+ return []byte("%2b")
+ case ',':
+ return []byte("%2c")
+ case '-':
+ return []byte("-") // bound without encoding
+ case '.':
+ return []byte(".") // bound without encoding
+ case '/':
+ return []byte("%2f")
+ case ':':
+ return []byte("%3a")
+ case ';':
+ return []byte("%3b")
+ case '<':
+ return []byte("%3c")
+ case '=':
+ return []byte("%3d")
+ case '>':
+ return []byte("%3e")
+ case '?':
+ return []byte("%3f")
+ case '@':
+ return []byte("%40")
+ case '[':
+ return []byte("%5b")
+ case '\\':
+ return []byte("%5c")
+ case ']':
+ return []byte("%5d")
+ case '^':
+ return []byte("%5e")
+ case '`':
+ return []byte("%60")
+ case '{':
+ return []byte("%7b")
+ case '|':
+ return []byte("%7c")
+ case '}':
+ return []byte("%7d")
+ case '~':
+ return []byte("%7e")
+ default:
+ return []byte{b}
+ }
+}
diff --git a/server/vulnerabilities/nvd/tools/wfn/uri_test.go b/server/vulnerabilities/nvd/tools/wfn/uri_test.go
new file mode 100644
index 0000000000..ddb33d1181
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/wfn/uri_test.go
@@ -0,0 +1,110 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package wfn
+
+import (
+ "fmt"
+ "testing"
+)
+
+func TestUnbindURI(t *testing.T) {
+ cases := []struct {
+ URI string
+ Expect string
+ Fail bool
+ }{
+ {
+ URI: "cpe:/a",
+ Expect: `wfn:[part="a",vendor=ANY,product=ANY,version=ANY,update=ANY,edition=ANY,language=ANY]`,
+ },
+ {
+ URI: "cpe:/a:microsoft:internet_explorer:8.0.6001:beta",
+ Expect: `wfn:[part="a",vendor="microsoft",product="internet_explorer",version="8\.0\.6001",update="beta",edition=ANY,language=ANY]`,
+ },
+ {
+ URI: "cpe:/a:microsoft:internet_explorer:8.%2a:sp%3f",
+ Expect: `wfn:[part="a",vendor="microsoft",product="internet_explorer",version="8\.\*",update="sp\?",edition=ANY,language=ANY]`,
+ },
+ {
+ URI: "cpe:/a:microsoft:internet_explorer:8.%02:sp%01",
+ Expect: `wfn:[part="a",vendor="microsoft",product="internet_explorer",version="8\.*",update="sp?",edition=ANY,language=ANY]`,
+ },
+ {
+ URI: "cpe:/a:Microsoft:internet_explorer:8.%02:sp%01:limited",
+ Expect: `wfn:[part="a",vendor="microsoft",product="internet_explorer",version="8\.*",update="sp?",edition="limited",language=ANY]`,
+ },
+ {
+ URI: "cpe:/a:hp:insight_diagnostics:7.4.0.1570::~~online~win2003~x64~",
+ Expect: `wfn:[part="a",vendor="hp",product="insight_diagnostics",version="7\.4\.0\.1570",update=ANY,edition=ANY,sw_edition="online",target_sw="win2003",target_hw="x64",other=ANY,language=ANY]`,
+ },
+ {
+ URI: "cpe:/o:microsoft:windows_10:-::~~~~x64~",
+ Expect: `wfn:[part="o",vendor="microsoft",product="windows_10",version=NA,update=ANY,edition=ANY,sw_edition=ANY,target_sw=ANY,target_hw="x64",other=ANY,language=ANY]`,
+ },
+ {
+ URI: `cpe:/a:foo:boo%02%02`,
+ Fail: true,
+ },
+ {
+ URI: "cpe:/a:foo:bar:12.%02.1234",
+ Fail: true,
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.URI, func(t *testing.T) {
+ attr, err := UnbindURI(tc.URI)
+ if err != nil {
+ if tc.Fail {
+ return
+ }
+ t.Fatalf("failed to parse URI %q: %v", tc.URI, err)
+ }
+ if tc.Fail {
+ t.Fatalf("URI parsed successfully, despite failure was expected: %q", tc.URI)
+ }
+ if attr.String() != tc.Expect {
+ t.Fatalf("expected %s\ngot %s", tc.Expect, attr)
+ }
+ })
+ }
+}
+
+func BenchmarkUnbindURI(t *testing.B) {
+ for i := 0; i < t.N; i++ {
+ _, _ = UnbindURI("cpe:/a:hp:insight_diagnostics:7.4.0.1570::~~online~win2003~x64~")
+ }
+}
+
+func TestBindToURI(t *testing.T) {
+ cases := []string{
+ "cpe:/a:microsoft:internet_explorer:8.0.6001:beta",
+ "cpe:/a:microsoft:internet_explorer:8.%2a:sp%3f",
+ "cpe:/a:microsoft:internet_explorer:8.%02:sp%01",
+ "cpe:/a:microsoft:internet_explorer:8.%02:sp%01:limited",
+ "cpe:/a:hp:insight_diagnostics:7.4.0.1570::~~online~win2003~x64~",
+ }
+ for n, c := range cases {
+ c := c
+ t.Run(fmt.Sprintf("case#%d", n), func(t *testing.T) {
+ attr, err := UnbindURI(c)
+ if err != nil {
+ t.Fatalf("failed to parse input %q: %v", c, err)
+ }
+ if out := attr.BindToURI(); out != c {
+ t.Fatalf("expected %s\ngot %s", c, out)
+ }
+ })
+ }
+}
diff --git a/server/vulnerabilities/nvd/tools/wfn/wfn.go b/server/vulnerabilities/nvd/tools/wfn/wfn.go
new file mode 100644
index 0000000000..0cce686050
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/wfn/wfn.go
@@ -0,0 +1,163 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package wfn
+
+import (
+ "fmt"
+ "strings"
+)
+
+// KnownParts is a map of known WFN attribute parts.
+var KnownParts = map[string]string{
+ "a": "application",
+ "o": "operating system",
+ "h": "hardware",
+}
+
+// Possible logical value of Attributes
+// empty string considered ANY when parsing and unquoted "-" is illegal in WFN attribute-value
+const (
+ Any = ""
+ NA = "-"
+)
+
+const (
+ uriPrefix = "cpe:/"
+ fsbPrefix = "cpe:2.3:"
+)
+
+var parsers = map[string]func(s string) (*Attributes, error){
+ uriPrefix: UnbindURI,
+ fsbPrefix: UnbindFmtString,
+}
+
+// Parse parses Attributes from URI or formatted string binding.
+func Parse(s string) (*Attributes, error) {
+ for prefix, parserFunc := range parsers {
+ if strings.HasPrefix(s, prefix) {
+ return parserFunc(s)
+ }
+ }
+ return nil, fmt.Errorf("wfn: unsupported format %q", s)
+}
+
+// Attributes defines the WFN Data Model Attributes.
+type Attributes struct {
+ Part string
+ Vendor string
+ Product string
+ Version string
+ Update string
+ Edition string
+ SWEdition string
+ TargetSW string
+ TargetHW string
+ Other string
+ Language string
+}
+
+// NewAttributesWithNA allocates Attributes object with all fields initialized to NA logical value
+func NewAttributesWithNA() *Attributes {
+ return newAttributes(NA)
+}
+
+// NewAttributesWithAny allocates Attributes object with all fields initialized to Any logical value
+func NewAttributesWithAny() *Attributes {
+ return newAttributes(Any)
+}
+
+func newAttributes(defaultValue string) *Attributes {
+ return &Attributes{
+ Part: defaultValue,
+ Vendor: defaultValue,
+ Product: defaultValue,
+ Version: defaultValue,
+ Update: defaultValue,
+ Edition: defaultValue,
+ SWEdition: defaultValue,
+ TargetSW: defaultValue,
+ TargetHW: defaultValue,
+ Other: defaultValue,
+ Language: defaultValue,
+ }
+}
+
+// WFNize transforms a string into CPE23-NAME compliant avstring value.
+// This function isn't a part of standard. Quoted wildcards (*?) become unquoted ones (i.e. act as wildcards,
+// not a literal '*' and '?')
+// If wildcards are used, it is a responsibility of the user to make sure they comply with the standard, i.e.
+// only appear at the beginning or at the end of the string and, in case of asterisk, only once in each case.
+// Uppercase letters are valid avstring characters, but they are rarely (if ever) used in WFNs. It is recommended
+// to strings.ToLower() the string before passing it to this function.
+func WFNize(s string) (string, error) {
+ const allowedPunct = "-!\"#$%&'()+,./:;<=>@[]^`{|}!~"
+ // replace spaces with underscores
+ in := strings.Replace(s, " ", "_", -1)
+ buf := make([]byte, 0, len(in))
+ // remove illegal characters
+ for n, c := range in {
+ c := byte(c)
+ if c >= 'A' && c <= 'Z' ||
+ c >= 'a' && c <= 'z' ||
+ c >= '0' && c <= '9' ||
+ c == '_' ||
+ strings.IndexByte(allowedPunct, c) != -1 {
+ buf = append(buf, c)
+ }
+ // handle wildcard characters
+ if c == '*' || c == '?' {
+ if n == 0 || in[n-1] != '\\' {
+ buf = append(buf, '\\')
+ }
+ buf = append(buf, c)
+ }
+ }
+ // quote everything that requires quoting
+ s, _, err := addSlashesAt(string(buf), 0)
+ return s, err
+}
+
+// String returns a string representation of the wfn
+func (a Attributes) String() string {
+ parts := make([]string, 0, 11)
+ // these are always displayed
+ parts = append(parts, keyValueString("part", a.Part))
+ parts = append(parts, keyValueString("vendor", a.Vendor))
+ parts = append(parts, keyValueString("product", a.Product))
+ parts = append(parts, keyValueString("version", a.Version))
+ parts = append(parts, keyValueString("update", a.Update))
+ parts = append(parts, keyValueString("edition", a.Edition))
+ // these are present only if one of them isn't ANY (cpe:2.2 compartibility)
+ if a.SWEdition != Any || a.TargetHW != Any || a.TargetSW != Any || a.Other != Any {
+ parts = append(parts, keyValueString("sw_edition", a.SWEdition))
+ parts = append(parts, keyValueString("target_sw", a.TargetSW))
+ parts = append(parts, keyValueString("target_hw", a.TargetHW))
+ parts = append(parts, keyValueString("other", a.Other))
+ }
+ // also always displayed
+ parts = append(parts, keyValueString("language", a.Language))
+ return fmt.Sprintf("wfn:[%s]", strings.Join(parts, ","))
+}
+
+func keyValueString(k, v string) string {
+ switch v {
+ case Any:
+ return fmt.Sprintf("%s=ANY", k)
+ case NA:
+ return fmt.Sprintf("%s=NA", k)
+ default:
+ return fmt.Sprintf("%s=\"%s\"", k, v)
+ }
+}
diff --git a/server/vulnerabilities/nvd/tools/wfn/wfn_test.go b/server/vulnerabilities/nvd/tools/wfn/wfn_test.go
new file mode 100644
index 0000000000..63cd33450f
--- /dev/null
+++ b/server/vulnerabilities/nvd/tools/wfn/wfn_test.go
@@ -0,0 +1,52 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package wfn
+
+import (
+ "testing"
+)
+
+func TestWFNize(t *testing.T) {
+ cases := []struct {
+ in string
+ expected string
+ expectErr bool
+ }{
+ {"Zonealarm Wireless Security", "Zonealarm_Wireless_Security", false},
+ {"1.8.14.6001", `1\.8\.14\.6001`, false},
+ {"xorg-server", `xorg\-server`, false},
+ {`1.8.\*`, `1\.8\.*`, false},
+ {"1.*.14", `1\.\*\.14`, false},
+ {`1.\*.14`, "", true},
+ }
+ for _, c := range cases {
+ res, err := WFNize(c.in)
+ if err != nil {
+ if !c.expectErr {
+ t.Errorf("WFNize(%q) returned error: %v", c.in, err)
+ }
+ } else if c.expectErr {
+ t.Errorf("WFNize(%q) was expected to fail, but succedeed", c.in)
+ } else if res != c.expected {
+ t.Errorf("WFNize(%q) returned %q, %q was expected", c.in, res, c.expected)
+ }
+ }
+}
+
+func BenchmarkWFNize(t *testing.B) {
+ for i := 0; i < t.N; i++ {
+ _, _ = WFNize("1.8.14.6001")
+ }
+}
diff --git a/server/worker/macos_setup_assistant.go b/server/worker/macos_setup_assistant.go
index cf206c9ff5..3850adc641 100644
--- a/server/worker/macos_setup_assistant.go
+++ b/server/worker/macos_setup_assistant.go
@@ -384,7 +384,7 @@ func ProcessDEPCooldowns(ctx context.Context, ds fleet.Datastore, logger kitlog.
return ctxerr.Wrap(ctx, err, "getting cooldowns")
}
if len(serialsByTeamId) == 0 {
- logger.Log("msg", "no cooldowns to process")
+ level.Info(logger).Log("msg", "no cooldowns to process")
return nil
}
@@ -394,7 +394,7 @@ func ProcessDEPCooldowns(ctx context.Context, ds fleet.Datastore, logger kitlog.
logger.Log("msg", "no cooldowns", "team_id", teamID)
continue
}
- logger.Log("msg", "processing cooldowns", "team_id", teamID, "serials", serials)
+ level.Info(logger).Log("msg", "processing cooldowns", "team_id", teamID, "serials", serials)
var tid *uint
if teamID != 0 {
diff --git a/terraform/README.md b/terraform/README.md
index f50b23c952..cb1fd829a1 100644
--- a/terraform/README.md
+++ b/terraform/README.md
@@ -75,7 +75,7 @@ No resources.
| [alb\_config](#input\_alb\_config) | n/a | object({
name = optional(string, "fleet")
security_groups = optional(list(string), [])
access_logs = optional(map(string), {})
allowed_cidrs = optional(list(string), ["0.0.0.0/0"])
allowed_ipv6_cidrs = optional(list(string), ["::/0"])
egress_cidrs = optional(list(string), ["0.0.0.0/0"])
egress_ipv6_cidrs = optional(list(string), ["::/0"])
extra_target_groups = optional(any, [])
https_listener_rules = optional(any, [])
tls_policy = optional(string, "ELBSecurityPolicy-TLS-1-2-2017-01")
idle_timeout = optional(number, 60)
}) | `{}` | no |
| [certificate\_arn](#input\_certificate\_arn) | n/a | `string` | n/a | yes |
| [ecs\_cluster](#input\_ecs\_cluster) | The config for the terraform-aws-modules/ecs/aws module | object({
autoscaling_capacity_providers = optional(any, {})
cluster_configuration = optional(any, {
execute_command_configuration = {
logging = "OVERRIDE"
log_configuration = {
cloud_watch_log_group_name = "/aws/ecs/aws-ec2"
}
}
})
cluster_name = optional(string, "fleet")
cluster_settings = optional(map(string), {
"name" : "containerInsights",
"value" : "enabled",
})
create = optional(bool, true)
default_capacity_provider_use_fargate = optional(bool, true)
fargate_capacity_providers = optional(any, {
FARGATE = {
default_capacity_provider_strategy = {
weight = 100
}
}
FARGATE_SPOT = {
default_capacity_provider_strategy = {
weight = 0
}
}
})
tags = optional(map(string))
}) | {
"autoscaling_capacity_providers": {},
"cluster_configuration": {
"execute_command_configuration": {
"log_configuration": {
"cloud_watch_log_group_name": "/aws/ecs/aws-ec2"
},
"logging": "OVERRIDE"
}
},
"cluster_name": "fleet",
"cluster_settings": {
"name": "containerInsights",
"value": "enabled"
},
"create": true,
"default_capacity_provider_use_fargate": true,
"fargate_capacity_providers": {
"FARGATE": {
"default_capacity_provider_strategy": {
"weight": 100
}
},
"FARGATE_SPOT": {
"default_capacity_provider_strategy": {
"weight": 0
}
}
},
"tags": {}
} | no |
-| [fleet\_config](#input\_fleet\_config) | The configuration object for Fleet itself. Fields that default to null will have their respective resources created if not specified. | object({
mem = optional(number, 4096)
cpu = optional(number, 512)
image = optional(string, "fleetdm/fleet:v4.48.3")
family = optional(string, "fleet")
sidecars = optional(list(any), [])
depends_on = optional(list(any), [])
mount_points = optional(list(any), [])
volumes = optional(list(any), [])
extra_environment_variables = optional(map(string), {})
extra_iam_policies = optional(list(string), [])
extra_execution_iam_policies = optional(list(string), [])
extra_secrets = optional(map(string), {})
security_groups = optional(list(string), null)
security_group_name = optional(string, "fleet")
iam_role_arn = optional(string, null)
repository_credentials = optional(string, "")
service = optional(object({
name = optional(string, "fleet")
}), {
name = "fleet"
})
database = optional(object({
password_secret_arn = string
user = string
database = string
address = string
rr_address = optional(string, null)
}), {
password_secret_arn = null
user = null
database = null
address = null
rr_address = null
})
redis = optional(object({
address = string
use_tls = optional(bool, true)
}), {
address = null
use_tls = true
})
awslogs = optional(object({
name = optional(string, null)
region = optional(string, null)
create = optional(bool, true)
prefix = optional(string, "fleet")
retention = optional(number, 5)
}), {
name = null
region = null
prefix = "fleet"
retention = 5
})
loadbalancer = optional(object({
arn = string
}), {
arn = null
})
extra_load_balancers = optional(list(any), [])
networking = optional(object({
subnets = list(string)
security_groups = optional(list(string), null)
}), {
subnets = null
security_groups = null
})
autoscaling = optional(object({
max_capacity = optional(number, 5)
min_capacity = optional(number, 1)
memory_tracking_target_value = optional(number, 80)
cpu_tracking_target_value = optional(number, 80)
}), {
max_capacity = 5
min_capacity = 1
memory_tracking_target_value = 80
cpu_tracking_target_value = 80
})
iam = optional(object({
role = optional(object({
name = optional(string, "fleet-role")
policy_name = optional(string, "fleet-iam-policy")
}), {
name = "fleet-role"
policy_name = "fleet-iam-policy"
})
execution = optional(object({
name = optional(string, "fleet-execution-role")
policy_name = optional(string, "fleet-execution-role")
}), {
name = "fleet-execution-role"
policy_name = "fleet-iam-policy-execution"
})
}), {
name = "fleetdm-execution-role"
})
}) | {
"autoscaling": {
"cpu_tracking_target_value": 80,
"max_capacity": 5,
"memory_tracking_target_value": 80,
"min_capacity": 1
},
"awslogs": {
"create": true,
"name": null,
"prefix": "fleet",
"region": null,
"retention": 5
},
"cpu": 256,
"database": {
"address": null,
"database": null,
"password_secret_arn": null,
"rr_address": null,
"user": null
},
"depends_on": [],
"extra_environment_variables": {},
"extra_execution_iam_policies": [],
"extra_iam_policies": [],
"extra_load_balancers": [],
"extra_secrets": {},
"family": "fleet",
"iam": {
"execution": {
"name": "fleet-execution-role",
"policy_name": "fleet-iam-policy-execution"
},
"role": {
"name": "fleet-role",
"policy_name": "fleet-iam-policy"
}
},
"iam_role_arn": null,
"image": "fleetdm/fleet:v4.31.1",
"loadbalancer": {
"arn": null
},
"mem": 512,
"mount_points": [],
"networking": {
"security_groups": null,
"subnets": null
},
"redis": {
"address": null,
"use_tls": true
},
"repository_credentials": "",
"security_group_name": "fleet",
"security_groups": null,
"service": {
"name": "fleet"
},
"sidecars": [],
"volumes": []
} | no |
+| [fleet\_config](#input\_fleet\_config) | The configuration object for Fleet itself. Fields that default to null will have their respective resources created if not specified. | object({
mem = optional(number, 4096)
cpu = optional(number, 512)
image = optional(string, "fleetdm/fleet:v4.49.1")
family = optional(string, "fleet")
sidecars = optional(list(any), [])
depends_on = optional(list(any), [])
mount_points = optional(list(any), [])
volumes = optional(list(any), [])
extra_environment_variables = optional(map(string), {})
extra_iam_policies = optional(list(string), [])
extra_execution_iam_policies = optional(list(string), [])
extra_secrets = optional(map(string), {})
security_groups = optional(list(string), null)
security_group_name = optional(string, "fleet")
iam_role_arn = optional(string, null)
repository_credentials = optional(string, "")
service = optional(object({
name = optional(string, "fleet")
}), {
name = "fleet"
})
database = optional(object({
password_secret_arn = string
user = string
database = string
address = string
rr_address = optional(string, null)
}), {
password_secret_arn = null
user = null
database = null
address = null
rr_address = null
})
redis = optional(object({
address = string
use_tls = optional(bool, true)
}), {
address = null
use_tls = true
})
awslogs = optional(object({
name = optional(string, null)
region = optional(string, null)
create = optional(bool, true)
prefix = optional(string, "fleet")
retention = optional(number, 5)
}), {
name = null
region = null
prefix = "fleet"
retention = 5
})
loadbalancer = optional(object({
arn = string
}), {
arn = null
})
extra_load_balancers = optional(list(any), [])
networking = optional(object({
subnets = list(string)
security_groups = optional(list(string), null)
}), {
subnets = null
security_groups = null
})
autoscaling = optional(object({
max_capacity = optional(number, 5)
min_capacity = optional(number, 1)
memory_tracking_target_value = optional(number, 80)
cpu_tracking_target_value = optional(number, 80)
}), {
max_capacity = 5
min_capacity = 1
memory_tracking_target_value = 80
cpu_tracking_target_value = 80
})
iam = optional(object({
role = optional(object({
name = optional(string, "fleet-role")
policy_name = optional(string, "fleet-iam-policy")
}), {
name = "fleet-role"
policy_name = "fleet-iam-policy"
})
execution = optional(object({
name = optional(string, "fleet-execution-role")
policy_name = optional(string, "fleet-execution-role")
}), {
name = "fleet-execution-role"
policy_name = "fleet-iam-policy-execution"
})
}), {
name = "fleetdm-execution-role"
})
}) | {
"autoscaling": {
"cpu_tracking_target_value": 80,
"max_capacity": 5,
"memory_tracking_target_value": 80,
"min_capacity": 1
},
"awslogs": {
"create": true,
"name": null,
"prefix": "fleet",
"region": null,
"retention": 5
},
"cpu": 256,
"database": {
"address": null,
"database": null,
"password_secret_arn": null,
"rr_address": null,
"user": null
},
"depends_on": [],
"extra_environment_variables": {},
"extra_execution_iam_policies": [],
"extra_iam_policies": [],
"extra_load_balancers": [],
"extra_secrets": {},
"family": "fleet",
"iam": {
"execution": {
"name": "fleet-execution-role",
"policy_name": "fleet-iam-policy-execution"
},
"role": {
"name": "fleet-role",
"policy_name": "fleet-iam-policy"
}
},
"iam_role_arn": null,
"image": "fleetdm/fleet:v4.31.1",
"loadbalancer": {
"arn": null
},
"mem": 512,
"mount_points": [],
"networking": {
"security_groups": null,
"subnets": null
},
"redis": {
"address": null,
"use_tls": true
},
"repository_credentials": "",
"security_group_name": "fleet",
"security_groups": null,
"service": {
"name": "fleet"
},
"sidecars": [],
"volumes": []
} | no |
| [migration\_config](#input\_migration\_config) | The configuration object for Fleet's migration task. | object({
mem = number
cpu = number
}) | {
"cpu": 1024,
"mem": 2048
} | no |
| [rds\_config](#input\_rds\_config) | The config for the terraform-aws-modules/rds-aurora/aws module | object({
name = optional(string, "fleet")
engine_version = optional(string, "8.0.mysql_aurora.3.04.2")
instance_class = optional(string, "db.t4g.large")
subnets = optional(list(string), [])
allowed_security_groups = optional(list(string), [])
allowed_cidr_blocks = optional(list(string), [])
apply_immediately = optional(bool, true)
monitoring_interval = optional(number, 10)
db_parameter_group_name = optional(string)
db_parameters = optional(map(string), {})
db_cluster_parameter_group_name = optional(string)
db_cluster_parameters = optional(map(string), {})
enabled_cloudwatch_logs_exports = optional(list(string), [])
master_username = optional(string, "fleet")
snapshot_identifier = optional(string)
cluster_tags = optional(map(string), {})
}) | {
"allowed_cidr_blocks": [],
"allowed_security_groups": [],
"apply_immediately": true,
"cluster_tags": {},
"db_cluster_parameter_group_name": null,
"db_cluster_parameters": {},
"db_parameter_group_name": null,
"db_parameters": {},
"enabled_cloudwatch_logs_exports": [],
"engine_version": "8.0.mysql_aurora.3.04.2",
"instance_class": "db.t4g.large",
"master_username": "fleet",
"monitoring_interval": 10,
"name": "fleet",
"snapshot_identifier": null,
"subnets": []
} | no |
| [redis\_config](#input\_redis\_config) | n/a | object({
name = optional(string, "fleet")
replication_group_id = optional(string)
elasticache_subnet_group_name = optional(string)
allowed_security_group_ids = optional(list(string), [])
subnets = optional(list(string))
availability_zones = optional(list(string))
cluster_size = optional(number, 3)
instance_type = optional(string, "cache.m5.large")
apply_immediately = optional(bool, true)
automatic_failover_enabled = optional(bool, false)
engine_version = optional(string, "6.x")
family = optional(string, "redis6.x")
at_rest_encryption_enabled = optional(bool, true)
transit_encryption_enabled = optional(bool, true)
parameter = optional(list(object({
name = string
value = string
})), [])
log_delivery_configuration = optional(list(map(any)), [])
tags = optional(map(string), {})
}) | {
"allowed_security_group_ids": [],
"apply_immediately": true,
"at_rest_encryption_enabled": true,
"automatic_failover_enabled": false,
"availability_zones": null,
"cluster_size": 3,
"elasticache_subnet_group_name": null,
"engine_version": "6.x",
"family": "redis6.x",
"instance_type": "cache.m5.large",
"log_delivery_configuration": [],
"name": "fleet",
"parameter": [],
"replication_group_id": null,
"subnets": null,
"tags": {},
"transit_encryption_enabled": true
} | no |
diff --git a/terraform/byo-vpc/README.md b/terraform/byo-vpc/README.md
index 651d206147..89aa013d1f 100644
--- a/terraform/byo-vpc/README.md
+++ b/terraform/byo-vpc/README.md
@@ -34,7 +34,7 @@ No requirements.
| [alb\_config](#input\_alb\_config) | n/a | object({
name = optional(string, "fleet")
subnets = list(string)
security_groups = optional(list(string), [])
access_logs = optional(map(string), {})
certificate_arn = string
allowed_cidrs = optional(list(string), ["0.0.0.0/0"])
allowed_ipv6_cidrs = optional(list(string), ["::/0"])
egress_cidrs = optional(list(string), ["0.0.0.0/0"])
egress_ipv6_cidrs = optional(list(string), ["::/0"])
extra_target_groups = optional(any, [])
https_listener_rules = optional(any, [])
tls_policy = optional(string, "ELBSecurityPolicy-TLS-1-2-2017-01")
idle_timeout = optional(number, 60)
}) | n/a | yes |
| [ecs\_cluster](#input\_ecs\_cluster) | The config for the terraform-aws-modules/ecs/aws module | object({
autoscaling_capacity_providers = optional(any, {})
cluster_configuration = optional(any, {
execute_command_configuration = {
logging = "OVERRIDE"
log_configuration = {
cloud_watch_log_group_name = "/aws/ecs/aws-ec2"
}
}
})
cluster_name = optional(string, "fleet")
cluster_settings = optional(map(string), {
"name" : "containerInsights",
"value" : "enabled",
})
create = optional(bool, true)
default_capacity_provider_use_fargate = optional(bool, true)
fargate_capacity_providers = optional(any, {
FARGATE = {
default_capacity_provider_strategy = {
weight = 100
}
}
FARGATE_SPOT = {
default_capacity_provider_strategy = {
weight = 0
}
}
})
tags = optional(map(string))
}) | {
"autoscaling_capacity_providers": {},
"cluster_configuration": {
"execute_command_configuration": {
"log_configuration": {
"cloud_watch_log_group_name": "/aws/ecs/aws-ec2"
},
"logging": "OVERRIDE"
}
},
"cluster_name": "fleet",
"cluster_settings": {
"name": "containerInsights",
"value": "enabled"
},
"create": true,
"default_capacity_provider_use_fargate": true,
"fargate_capacity_providers": {
"FARGATE": {
"default_capacity_provider_strategy": {
"weight": 100
}
},
"FARGATE_SPOT": {
"default_capacity_provider_strategy": {
"weight": 0
}
}
},
"tags": {}
} | no |
<<<<<<< HEAD
-| [fleet\_config](#input\_fleet\_config) | The configuration object for Fleet itself. Fields that default to null will have their respective resources created if not specified. | object({
mem = optional(number, 4096)
cpu = optional(number, 512)
image = optional(string, "fleetdm/fleet:v4.48.3")
family = optional(string, "fleet")
sidecars = optional(list(any), [])
depends_on = optional(list(any), [])
mount_points = optional(list(any), [])
volumes = optional(list(any), [])
extra_environment_variables = optional(map(string), {})
extra_iam_policies = optional(list(string), [])
extra_execution_iam_policies = optional(list(string), [])
extra_secrets = optional(map(string), {})
security_groups = optional(list(string), null)
security_group_name = optional(string, "fleet")
iam_role_arn = optional(string, null)
service = optional(object({
name = optional(string, "fleet")
}), {
name = "fleet"
})
database = optional(object({
password_secret_arn = string
user = string
database = string
address = string
rr_address = optional(string, null)
}), {
password_secret_arn = null
user = null
database = null
address = null
rr_address = null
})
redis = optional(object({
address = string
use_tls = optional(bool, true)
}), {
address = null
use_tls = true
})
awslogs = optional(object({
name = optional(string, null)
region = optional(string, null)
create = optional(bool, true)
prefix = optional(string, "fleet")
retention = optional(number, 5)
}), {
name = null
region = null
prefix = "fleet"
retention = 5
})
loadbalancer = optional(object({
arn = string
}), {
arn = null
})
extra_load_balancers = optional(list(any), [])
networking = optional(object({
subnets = list(string)
security_groups = optional(list(string), null)
}), {
subnets = null
security_groups = null
})
autoscaling = optional(object({
max_capacity = optional(number, 5)
min_capacity = optional(number, 1)
memory_tracking_target_value = optional(number, 80)
cpu_tracking_target_value = optional(number, 80)
}), {
max_capacity = 5
min_capacity = 1
memory_tracking_target_value = 80
cpu_tracking_target_value = 80
})
iam = optional(object({
role = optional(object({
name = optional(string, "fleet-role")
policy_name = optional(string, "fleet-iam-policy")
}), {
name = "fleet-role"
policy_name = "fleet-iam-policy"
})
execution = optional(object({
name = optional(string, "fleet-execution-role")
policy_name = optional(string, "fleet-execution-role")
}), {
name = "fleet-execution-role"
policy_name = "fleet-iam-policy-execution"
})
}), {
name = "fleetdm-execution-role"
})
}) | {
"autoscaling": {
"cpu_tracking_target_value": 80,
"max_capacity": 5,
"memory_tracking_target_value": 80,
"min_capacity": 1
},
"awslogs": {
"create": true,
"name": null,
"prefix": "fleet",
"region": null,
"retention": 5
},
"cpu": 256,
"database": {
"address": null,
"database": null,
"password_secret_arn": null,
"rr_address": null,
"user": null
},
"depends_on": [],
"extra_environment_variables": {},
"extra_execution_iam_policies": [],
"extra_iam_policies": [],
"extra_load_balancers": [],
"extra_secrets": {},
"family": "fleet",
"iam": {
"execution": {
"name": "fleet-execution-role",
"policy_name": "fleet-iam-policy-execution"
},
"role": {
"name": "fleet-role",
"policy_name": "fleet-iam-policy"
}
},
"iam_role_arn": null,
"image": "fleetdm/fleet:v4.31.1",
"loadbalancer": {
"arn": null
},
"mem": 512,
"mount_points": [],
"networking": {
"security_groups": null,
"subnets": null
},
"redis": {
"address": null,
"use_tls": true
},
"security_group_name": "fleet",
"security_groups": null,
"service": {
"name": "fleet"
},
"sidecars": [],
"volumes": []
} | no |
+| [fleet\_config](#input\_fleet\_config) | The configuration object for Fleet itself. Fields that default to null will have their respective resources created if not specified. | object({
mem = optional(number, 4096)
cpu = optional(number, 512)
image = optional(string, "fleetdm/fleet:v4.49.1")
family = optional(string, "fleet")
sidecars = optional(list(any), [])
depends_on = optional(list(any), [])
mount_points = optional(list(any), [])
volumes = optional(list(any), [])
extra_environment_variables = optional(map(string), {})
extra_iam_policies = optional(list(string), [])
extra_execution_iam_policies = optional(list(string), [])
extra_secrets = optional(map(string), {})
security_groups = optional(list(string), null)
security_group_name = optional(string, "fleet")
iam_role_arn = optional(string, null)
service = optional(object({
name = optional(string, "fleet")
}), {
name = "fleet"
})
database = optional(object({
password_secret_arn = string
user = string
database = string
address = string
rr_address = optional(string, null)
}), {
password_secret_arn = null
user = null
database = null
address = null
rr_address = null
})
redis = optional(object({
address = string
use_tls = optional(bool, true)
}), {
address = null
use_tls = true
})
awslogs = optional(object({
name = optional(string, null)
region = optional(string, null)
create = optional(bool, true)
prefix = optional(string, "fleet")
retention = optional(number, 5)
}), {
name = null
region = null
prefix = "fleet"
retention = 5
})
loadbalancer = optional(object({
arn = string
}), {
arn = null
})
extra_load_balancers = optional(list(any), [])
networking = optional(object({
subnets = list(string)
security_groups = optional(list(string), null)
}), {
subnets = null
security_groups = null
})
autoscaling = optional(object({
max_capacity = optional(number, 5)
min_capacity = optional(number, 1)
memory_tracking_target_value = optional(number, 80)
cpu_tracking_target_value = optional(number, 80)
}), {
max_capacity = 5
min_capacity = 1
memory_tracking_target_value = 80
cpu_tracking_target_value = 80
})
iam = optional(object({
role = optional(object({
name = optional(string, "fleet-role")
policy_name = optional(string, "fleet-iam-policy")
}), {
name = "fleet-role"
policy_name = "fleet-iam-policy"
})
execution = optional(object({
name = optional(string, "fleet-execution-role")
policy_name = optional(string, "fleet-execution-role")
}), {
name = "fleet-execution-role"
policy_name = "fleet-iam-policy-execution"
})
}), {
name = "fleetdm-execution-role"
})
}) | {
"autoscaling": {
"cpu_tracking_target_value": 80,
"max_capacity": 5,
"memory_tracking_target_value": 80,
"min_capacity": 1
},
"awslogs": {
"create": true,
"name": null,
"prefix": "fleet",
"region": null,
"retention": 5
},
"cpu": 256,
"database": {
"address": null,
"database": null,
"password_secret_arn": null,
"rr_address": null,
"user": null
},
"depends_on": [],
"extra_environment_variables": {},
"extra_execution_iam_policies": [],
"extra_iam_policies": [],
"extra_load_balancers": [],
"extra_secrets": {},
"family": "fleet",
"iam": {
"execution": {
"name": "fleet-execution-role",
"policy_name": "fleet-iam-policy-execution"
},
"role": {
"name": "fleet-role",
"policy_name": "fleet-iam-policy"
}
},
"iam_role_arn": null,
"image": "fleetdm/fleet:v4.31.1",
"loadbalancer": {
"arn": null
},
"mem": 512,
"mount_points": [],
"networking": {
"security_groups": null,
"subnets": null
},
"redis": {
"address": null,
"use_tls": true
},
"security_group_name": "fleet",
"security_groups": null,
"service": {
"name": "fleet"
},
"sidecars": [],
"volumes": []
} | no |
=======
| [fleet\_config](#input\_fleet\_config) | The configuration object for Fleet itself. Fields that default to null will have their respective resources created if not specified. | object({
mem = optional(number, 4096)
cpu = optional(number, 512)
image = optional(string, "fleetdm/fleet:v4.48.0")
family = optional(string, "fleet")
sidecars = optional(list(any), [])
depends_on = optional(list(any), [])
mount_points = optional(list(any), [])
volumes = optional(list(any), [])
extra_environment_variables = optional(map(string), {})
extra_iam_policies = optional(list(string), [])
extra_execution_iam_policies = optional(list(string), [])
extra_secrets = optional(map(string), {})
security_groups = optional(list(string), null)
security_group_name = optional(string, "fleet")
iam_role_arn = optional(string, null)
repository_credentials = optional(string, "")
service = optional(object({
name = optional(string, "fleet")
}), {
name = "fleet"
})
database = optional(object({
password_secret_arn = string
user = string
database = string
address = string
rr_address = optional(string, null)
}), {
password_secret_arn = null
user = null
database = null
address = null
rr_address = null
})
redis = optional(object({
address = string
use_tls = optional(bool, true)
}), {
address = null
use_tls = true
})
awslogs = optional(object({
name = optional(string, null)
region = optional(string, null)
create = optional(bool, true)
prefix = optional(string, "fleet")
retention = optional(number, 5)
}), {
name = null
region = null
prefix = "fleet"
retention = 5
})
loadbalancer = optional(object({
arn = string
}), {
arn = null
})
extra_load_balancers = optional(list(any), [])
networking = optional(object({
subnets = list(string)
security_groups = optional(list(string), null)
}), {
subnets = null
security_groups = null
})
autoscaling = optional(object({
max_capacity = optional(number, 5)
min_capacity = optional(number, 1)
memory_tracking_target_value = optional(number, 80)
cpu_tracking_target_value = optional(number, 80)
}), {
max_capacity = 5
min_capacity = 1
memory_tracking_target_value = 80
cpu_tracking_target_value = 80
})
iam = optional(object({
role = optional(object({
name = optional(string, "fleet-role")
policy_name = optional(string, "fleet-iam-policy")
}), {
name = "fleet-role"
policy_name = "fleet-iam-policy"
})
execution = optional(object({
name = optional(string, "fleet-execution-role")
policy_name = optional(string, "fleet-execution-role")
}), {
name = "fleet-execution-role"
policy_name = "fleet-iam-policy-execution"
})
}), {
name = "fleetdm-execution-role"
})
}) | {
"autoscaling": {
"cpu_tracking_target_value": 80,
"max_capacity": 5,
"memory_tracking_target_value": 80,
"min_capacity": 1
},
"awslogs": {
"create": true,
"name": null,
"prefix": "fleet",
"region": null,
"retention": 5
},
"cpu": 256,
"database": {
"address": null,
"database": null,
"password_secret_arn": null,
"rr_address": null,
"user": null
},
"depends_on": [],
"extra_environment_variables": {},
"extra_execution_iam_policies": [],
"extra_iam_policies": [],
"extra_load_balancers": [],
"extra_secrets": {},
"family": "fleet",
"iam": {
"execution": {
"name": "fleet-execution-role",
"policy_name": "fleet-iam-policy-execution"
},
"role": {
"name": "fleet-role",
"policy_name": "fleet-iam-policy"
}
},
"iam_role_arn": null,
"image": "fleetdm/fleet:v4.31.1",
"loadbalancer": {
"arn": null
},
"mem": 512,
"mount_points": [],
"networking": {
"security_groups": null,
"subnets": null
},
"redis": {
"address": null,
"use_tls": true
},
"repository_credentials": "",
"security_group_name": "fleet",
"security_groups": null,
"service": {
"name": "fleet"
},
"sidecars": [],
"volumes": []
} | no |
>>>>>>> 025004bcf (support private registry in the ecs task definition)
diff --git a/terraform/byo-vpc/byo-db/byo-ecs/variables.tf b/terraform/byo-vpc/byo-db/byo-ecs/variables.tf
index 907f153cb2..3e9bd4507a 100644
--- a/terraform/byo-vpc/byo-db/byo-ecs/variables.tf
+++ b/terraform/byo-vpc/byo-db/byo-ecs/variables.tf
@@ -13,7 +13,7 @@ variable "fleet_config" {
type = object({
mem = optional(number, 4096)
cpu = optional(number, 512)
- image = optional(string, "fleetdm/fleet:v4.48.3")
+ image = optional(string, "fleetdm/fleet:v4.49.1")
family = optional(string, "fleet")
sidecars = optional(list(any), [])
depends_on = optional(list(any), [])
diff --git a/terraform/byo-vpc/byo-db/variables.tf b/terraform/byo-vpc/byo-db/variables.tf
index ca169eebfa..194a11aef7 100644
--- a/terraform/byo-vpc/byo-db/variables.tf
+++ b/terraform/byo-vpc/byo-db/variables.tf
@@ -74,7 +74,7 @@ variable "fleet_config" {
type = object({
mem = optional(number, 4096)
cpu = optional(number, 512)
- image = optional(string, "fleetdm/fleet:v4.48.3")
+ image = optional(string, "fleetdm/fleet:v4.49.1")
family = optional(string, "fleet")
sidecars = optional(list(any), [])
depends_on = optional(list(any), [])
diff --git a/terraform/byo-vpc/example/main.tf b/terraform/byo-vpc/example/main.tf
index 897ec9ef89..aae37cc107 100644
--- a/terraform/byo-vpc/example/main.tf
+++ b/terraform/byo-vpc/example/main.tf
@@ -17,7 +17,7 @@ provider "aws" {
}
locals {
- fleet_image = "fleetdm/fleet:v4.48.3"
+ fleet_image = "fleetdm/fleet:v4.49.1"
domain_name = "example.com"
}
diff --git a/terraform/byo-vpc/variables.tf b/terraform/byo-vpc/variables.tf
index 66ed4ef168..e9463dda11 100644
--- a/terraform/byo-vpc/variables.tf
+++ b/terraform/byo-vpc/variables.tf
@@ -167,7 +167,7 @@ variable "fleet_config" {
type = object({
mem = optional(number, 4096)
cpu = optional(number, 512)
- image = optional(string, "fleetdm/fleet:v4.48.3")
+ image = optional(string, "fleetdm/fleet:v4.49.1")
family = optional(string, "fleet")
sidecars = optional(list(any), [])
depends_on = optional(list(any), [])
diff --git a/terraform/example/main.tf b/terraform/example/main.tf
index 9b1b314ff5..4a245a7969 100644
--- a/terraform/example/main.tf
+++ b/terraform/example/main.tf
@@ -59,8 +59,8 @@ module "fleet" {
fleet_config = {
# To avoid pull-rate limiting from dockerhub, consider using our quay.io mirror
- # for the Fleet image. e.g. "quay.io/fleetdm/fleet:v4.48.3"
- image = "fleetdm/fleet:v4.48.3" # override default to deploy the image you desire
+ # for the Fleet image. e.g. "quay.io/fleetdm/fleet:v4.49.1"
+ image = "fleetdm/fleet:v4.49.1" # override default to deploy the image you desire
# See https://fleetdm.com/docs/deploy/reference-architectures#aws for appropriate scaling
# memory and cpu.
autoscaling = {
diff --git a/terraform/variables.tf b/terraform/variables.tf
index d72f733ea1..d12ca9e1c7 100644
--- a/terraform/variables.tf
+++ b/terraform/variables.tf
@@ -215,7 +215,7 @@ variable "fleet_config" {
type = object({
mem = optional(number, 4096)
cpu = optional(number, 512)
- image = optional(string, "fleetdm/fleet:v4.48.3")
+ image = optional(string, "fleetdm/fleet:v4.49.1")
family = optional(string, "fleet")
sidecars = optional(list(any), [])
depends_on = optional(list(any), [])
diff --git a/tools/calendar/delete-events/delete-events.go b/tools/calendar/delete-events/delete-events.go
index 23933e11bf..dea8197dfa 100644
--- a/tools/calendar/delete-events/delete-events.go
+++ b/tools/calendar/delete-events/delete-events.go
@@ -27,7 +27,7 @@ var (
)
const (
- eventTitle = "💻🚫Downtime"
+ eventTitle = "💻🚫 Scheduled maintenance"
)
func main() {
diff --git a/tools/calendar/move-events/move-events.go b/tools/calendar/move-events/move-events.go
index d4906eaef1..7e413abaf7 100644
--- a/tools/calendar/move-events/move-events.go
+++ b/tools/calendar/move-events/move-events.go
@@ -28,7 +28,7 @@ var (
)
const (
- eventTitle = "💻🚫Downtime"
+ eventTitle = "💻🚫 Scheduled maintenance"
)
func main() {
diff --git a/tools/cloner-check/generated_files/appconfig.txt b/tools/cloner-check/generated_files/appconfig.txt
index 7fc10249d5..f3b6c8df0f 100644
--- a/tools/cloner-check/generated_files/appconfig.txt
+++ b/tools/cloner-check/generated_files/appconfig.txt
@@ -28,6 +28,9 @@ github.com/fleetdm/fleet/v4/server/fleet/SMTPSettings SMTPEnableStartTLS bool
github.com/fleetdm/fleet/v4/server/fleet/AppConfig HostExpirySettings fleet.HostExpirySettings
github.com/fleetdm/fleet/v4/server/fleet/HostExpirySettings HostExpiryEnabled bool
github.com/fleetdm/fleet/v4/server/fleet/HostExpirySettings HostExpiryWindow int
+github.com/fleetdm/fleet/v4/server/fleet/AppConfig ActivityExpirySettings fleet.ActivityExpirySettings
+github.com/fleetdm/fleet/v4/server/fleet/ActivityExpirySettings ActivityExpiryEnabled bool
+github.com/fleetdm/fleet/v4/server/fleet/ActivityExpirySettings ActivityExpiryWindow int
github.com/fleetdm/fleet/v4/server/fleet/AppConfig Features fleet.Features
github.com/fleetdm/fleet/v4/server/fleet/Features EnableHostUsers bool
github.com/fleetdm/fleet/v4/server/fleet/Features EnableSoftwareInventory bool
diff --git a/tools/fleetctl-npm/package.json b/tools/fleetctl-npm/package.json
index cc84162fe0..e14b40f2c6 100644
--- a/tools/fleetctl-npm/package.json
+++ b/tools/fleetctl-npm/package.json
@@ -1,6 +1,6 @@
{
"name": "fleetctl",
- "version": "v4.48.3",
+ "version": "v4.49.1",
"description": "Installer for the fleetctl CLI tool",
"bin": {
"fleetctl": "./run.js"
diff --git a/tools/mdm/apple/macos-vm-auto-enroll/README.md b/tools/mdm/apple/macos-vm-auto-enroll/README.md
index 7f50931750..3045544f71 100644
--- a/tools/mdm/apple/macos-vm-auto-enroll/README.md
+++ b/tools/mdm/apple/macos-vm-auto-enroll/README.md
@@ -9,6 +9,7 @@ The script takes no arguments, but can be configured through three environment v
- `FLEET_ENROLL_SECRET` (required) The fleet enrollment secret
- `FLEET_URL` (required) The fleet base url
- `MACOS_ENROLLMENT_VM_NAME` (optional) The name of the VM. If nothing is specified, the default name is `enrollment-test`.
+- `MACOS_ENROLLMENT_VM_IMAGE` (optional) The image to use for the VM. If nothing is specified, the default image is `ghcr.io/cirruslabs/macos-sonoma-base:latest`
The entire process from the generation of the `pkg` file to the installation is automated. The only part that requires user intervention is installing the MDM profile.
diff --git a/tools/mdm/apple/macos-vm-auto-enroll/macos-vm-auto-enroll.sh b/tools/mdm/apple/macos-vm-auto-enroll/macos-vm-auto-enroll.sh
index 32b469dd04..2d15275e75 100755
--- a/tools/mdm/apple/macos-vm-auto-enroll/macos-vm-auto-enroll.sh
+++ b/tools/mdm/apple/macos-vm-auto-enroll/macos-vm-auto-enroll.sh
@@ -6,9 +6,13 @@ set -m
# Fleet enroll secret placed in $FLEET_ENROLL_SECRET
# Fleet URL placed in $FLEET_URL
# Optional VM name in $MACOS_ENROLLMENT_VM_NAME
+# Optional VM image in $MACOS_ENROLLMENT_VM_IMAGE
+# For others see https://tart.run/quick-start/
+# - ghcr.io/cirruslabs/macos-ventura-base:latest
+# - ghcr.io/cirruslabs/macos-monterey-base:latest
vm_name="${MACOS_ENROLLMENT_VM_NAME:-enrollment-test}"
-image_name="ghcr.io/cirruslabs/macos-sonoma-base:latest"
+image_name="${MACOS_ENROLLMENT_VM_IMAGE:-ghcr.io/cirruslabs/macos-sonoma-base:latest}"
alias ssh_cmd="sshpass -p admin ssh -o \"StrictHostKeyChecking no\" admin@\$(tart ip $vm_name)"
alias ssh_interactive_cmd="sshpass -p admin ssh -o \"StrictHostKeyChecking no\" -t admin@\$(tart ip $vm_name)"
diff --git a/tools/release/README.md b/tools/release/README.md
index 08aed8a25d..0fe38b990a 100644
--- a/tools/release/README.md
+++ b/tools/release/README.md
@@ -39,7 +39,7 @@ example
# Tag main
./tools/release/publish_release.sh -ag
# Publish main
-./tools/release/publish_release.sh -au
+./tools/release/publish_release.sh -auq
# Go update osquery-slack version
```
diff --git a/tools/release/publish_release.sh b/tools/release/publish_release.sh
index 059b1595d0..1ea01e37ac 100755
--- a/tools/release/publish_release.sh
+++ b/tools/release/publish_release.sh
@@ -76,7 +76,8 @@ usage() {
echo " -f, --force Skip all confirmations"
echo " -h, --help Display this help message and exit"
echo " -g, --tag Run the tag step"
- echo " -m, --minor Increment to a minor version instead of patch (Required if including non-bugs"
+ echo " -m, --minor Increment to a minor version instead of patch (Required if including non-bugs)"
+ echo " -n, --announce_only Announce the release only, do not publish the release."
echo " -o, --open_api_key Set the Open API key for calling out to ChatGPT"
echo " -p, --print If the release is already drafted then print out the helpful info"
echo " -q, --quiet This will skip notifying in slack"
@@ -196,7 +197,7 @@ build_changelog() {
prompt=$'I am creating a changelog for an open source project from a list of commit messages. Please format it for me using the following rules:\n1. Correct spelling and punctuation.\n2. Sentence casing.\n3. Past tense.\n4. Each list item is designated with an asterisk.\n5. Output in markdown format.'
if [[ "$main_release" == "true" ]]; then
# Place to make a main targeted prompt
- prompt=$'I am creating a changelog for an open source project from a list of commit messages. Please format it for me using the following rules:\n1. Correct spelling and punctuation.\n2. Sentence casing.\n3. Past tense.\n4. Each list item is designated with an asterisk.\n5. Output in markdown format.'
+ prompt=$'I am creating a changelog for an open source project from a list of commit messages. Please format it for me using the following rules: Organize updates into three categories: Endpoint Operations, Device Management (MDM), and Vulnerability Management, with all bug fixes and misc. improvements listed under "Bug fixes and improvements". Start each entry with a past tense verb, using hyphens for bullet points. Include specific details for new features, bug fixes, API changes, and any necessary user actions. Note changes in user interfaces, system feedback, and significant architectural updates. Highlight mandatory actions and major impacts, especially for system administrators. Order seemingly important features at the top of their respective lists.'
fi
content=$(cat new_changelog | sed -E ':a;N;$!ba;s/\r{0,1}\n/\\n/g')
@@ -325,6 +326,44 @@ print_announce_info() {
fi
}
+general_announce_info() {
+ if [[ "$main_release" == "true" ]]; then
+ article_url="https://fleetdm.com/releases/fleet-$target_milestone"
+ article_published=`curl -is "$article_url" | head -n 1 | awk '{print $2}'`
+ if [[ "$article_published" != "200" ]]; then
+ echo "Could't find article at '$article_url'"
+ exit 1
+ fi
+
+ # TODO Publish Linkedin post about release article here and save url
+ linkedin_post_url=""
+ fi
+ echo "========================================================================="
+ echo "Update osquery Slack Fleet channel topic to say the correct version $next_ver"
+ echo "========================================================================="
+ # Slack
+ slack_hook_url=https://hooks.slack.com/services
+ app_id=T019PP37ALW
+ announce_text=":cloud: :rocket: The latest version of Fleet is $target_milestone.\nMore info: https://github.com/fleetdm/fleet/releases/tag/$next_tag"
+ if [[ "$main_release" == "true" ]]; then
+ announce_text=":cloud: :rocket: The latest version of Fleet is $target_milestone.\nMore info: https://github.com/fleetdm/fleet/releases/tag/$next_tag\nRelease article: $article_url\nLinkedIn post: $linkedin_post_url"
+ fi
+
+ echo -e $announce_text
+
+ if [ "$quiet" = "false" ]; then
+ if [ "$dry_run" = "false" ]; then
+ curl -X POST -H 'Content-type: application/json' \
+ --data "{\"text\":\"$announce_text\"}" \
+ $slack_hook_url/$app_id/$SLACK_GENERAL_TOKEN
+
+ curl -X POST -H 'Content -type: application/json' \
+ --data "{\"text\":\"$announce_text\nDogfood Deployed $dogfood_deploy\"}" \
+ $slack_hook_url/$app_id/$SLACK_HELP_INFRA_TOKEN
+ fi
+ fi
+}
+
update_release_notes() {
if [ "$dry_run" = "false" ]; then
if [ ! -f temp_changelog ]; then
@@ -412,62 +451,34 @@ tag() {
publish() {
if [ "$dry_run" = "false" ]; then
- if [[ "$main_release" == "true" ]]; then
- article_url="https://fleetdm.com/releases/fleet-$target_milestone"
- article_published=`curl -is "$article_url" | head -n 1 | awk '{print $2}'`
- if [[ "$article_published" != "200" ]]; then
- echo "Coulndn't find article at '$article_url'"
- exit 1
- fi
+ if [ "$announce_only" = "false" ]; then
+ # TODO more checks to validate we are ready to publish
+ gh release edit --draft=false --latest $next_tag
+ gh workflow run dogfood-deploy.yml -f DOCKER_IMAGE=fleetdm/fleet:$next_ver
+ show_spinner 200
+ dogfood_deploy=`gh run list --workflow=dogfood-deploy.yml --status in_progress -L 1 --json url | jq -r '.[] | .url'`
+ cd tools/fleetctl-npm && npm publish
- # TODO Publish Linkedin post about release article here and save url
- linkedin_post_url=""
- fi
- # TODO more checks to validate we are ready to publish
- gh release edit --draft=false --latest $next_tag
- gh workflow run dogfood-deploy.yml -f DOCKER_IMAGE=fleetdm/fleet:$next_ver
- show_spinner 200
- echo "========================================================================="
- echo "Update osquery Slack Fleet channel topic to say the correct version $next_ver"
- echo "========================================================================="
- dogfood_deploy=`gh run list --workflow=dogfood-deploy.yml --status in_progress -L 1 --json url | jq -r '.[] | .url'`
- cd tools/fleetctl-npm && npm publish
+ issues=`gh issue list -m $target_milestone --json number | jq -r '.[] | .number'`
+ for iss in $issues; do
+ is_story=`gh issue view $iss --json labels | jq -r '.labels | .[] | .name' | grep story`
+ # close all non-stories
+ if [[ "$is_story" == "" ]]; then
+ echo "Closing #$iss"
+ gh issue close $iss
+ fi
+ done
- issues=`gh issue list -m $target_milestone --json number | jq -r '.[] | .number'`
- for iss in $issues; do
- is_story=`gh issue view $iss --json labels | jq -r '.labels | .[] | .name' | grep story`
- # close all non-stories
- if [[ "$is_story" == "" ]]; then
- echo "Closing #$iss"
- gh issue close $iss
- fi
- done
-
- echo "Closing milestone"
- gh api repos/fleetdm/fleet/milestones/$target_milestone_number -f state=closed
-
- # Slack
- slack_hook_url=https://hooks.slack.com/services
- app_id=T019PP37ALW
- announce_text=":cloud: :rocket: The latest version of Fleet is $target_milestone.\nMore info: https://github.com/fleetdm/fleet/releases/tag/$next_tag\nUpgrade now: https://fleetdm.com/docs/deploying/upgrading-fleet"
- if [[ "$main_release" == "true" ]]; then
- announce_text=":cloud: :rocket: The latest version of Fleet is $target_milestone.\nMore info: https://github.com/fleetdm/fleet/releases/tag/$next_tag\nUpgrade now: https://fleetdm.com/docs/deploying/upgrading-fleet\nRelease Article: $article_url\nLinkedIn Post: $linkedin_post_url"
- fi
-
- echo $announce_text
-
- if [ "$quiet" = "false" ]; then
- curl -X POST -H 'Content-type: application/json' \
- --data "{\"text\":\"$announce_text\"}" \
- $slack_hook_url/$app_id/$SLACK_GENERAL_TOKEN
-
- curl -X POST -H 'Content-type: application/json' \
- --data "{\"text\":\"$announce_text\nDogfood Deployed $dogfood_deploy\"}" \
- $slack_hook_url/$app_id/$SLACK_HELP_INFRA_TOKEN
+ echo "Closing milestone"
+ gh api repos/fleetdm/fleet/milestones/$target_milestone_number -f state=closed
fi
else
echo "DRYRUN: Would have published $next_tag / deployed to dogfood / closed non-stories / closed milestone / announced in slack"
fi
+
+ echo "Send general announce"
+ # Send general announcement in #general
+ general_announce_info
}
# Validate we have all commands required to perform this script
@@ -478,6 +489,7 @@ cherry_pick_resolved=false
dry_run=false
force=false
minor=false
+announce_only=false
open_api_key=""
start_version=""
target_date=""
@@ -499,6 +511,7 @@ for arg in "$@"; do
"--force") set -- "$@" "-f" ;;
"--help") set -- "$@" "-h" ;;
"--minor") set -- "$@" "-m" ;;
+ "--announce_only") set -- "$@" "-n" ;;
"--open_api_key") set -- "$@" "-o" ;;
"--print") set -- "$@" "-p" ;;
"--quiet") set -- "$@" "-q" ;;
@@ -513,7 +526,7 @@ for arg in "$@"; do
done
# Extract options and their arguments using getopts
-while getopts "acdfhgmo:pqrs:t:uv:" opt; do
+while getopts "acdfhgmno:pqrs:t:uv:" opt; do
case "$opt" in
a) main_release=true ;;
c) cherry_pick_resolved=true ;;
@@ -522,6 +535,7 @@ while getopts "acdfhgmo:pqrs:t:uv:" opt; do
h) usage; exit 0 ;;
g) do_tag=true ;;
m) minor=true ;;
+ n) announce_only=true ;;
o) open_api_key=$OPTARG ;;
p) print_info=true ;;
q) quiet=true ;;
@@ -664,7 +678,6 @@ if [[ "$target_milestone_number" == "" ]]; then
fi
echo "Found milestone $target_milestone with number $target_milestone_number"
-
if [ "$print_info" = "true" ]; then
print_announce_info
exit 0
@@ -842,6 +855,13 @@ if [[ "$failed" == "false" ]]; then
echo "DRYRUN: Would have switched back to branch $target_patch_branch"
fi
+ if [[ "$main_release" == "false" ]]; then
+ # Cherry-pick from update-changelog-branch
+ ch_commit=`git log -n 1 --pretty=format:"%H" $update_changelog_branch`
+ git cherry-pick $ch_commit
+ git push origin $target_patch_branch -f
+ fi
+
# Check for QA issue
create_qa_issue
diff --git a/website/api/controllers/deliver-contact-form-message.js b/website/api/controllers/deliver-contact-form-message.js
index a746c93627..532308baf2 100644
--- a/website/api/controllers/deliver-contact-form-message.js
+++ b/website/api/controllers/deliver-contact-form-message.js
@@ -71,6 +71,12 @@ module.exports = {
`Name: ${firstName + ' ' + lastName}, Email: ${emailAddress}, Message: ${message ? message : 'No message.'}`
});
+ await sails.helpers.salesforce.updateOrCreateContactAndAccount.with({
+ emailAddress: emailAddress,
+ firstName: firstName,
+ lastName: lastName,
+ });
+
// Send a POST request to Zapier
await sails.helpers.http.post(
diff --git a/website/api/controllers/deliver-mdm-beta-signup.js b/website/api/controllers/deliver-mdm-beta-signup.js
index 5965858d9d..78eb54780b 100644
--- a/website/api/controllers/deliver-mdm-beta-signup.js
+++ b/website/api/controllers/deliver-mdm-beta-signup.js
@@ -1,6 +1,6 @@
module.exports = {
-
+ // TODO: This isn't a thing anymore, we can delete it.
friendlyName: 'Deliver MDM beta signup',
diff --git a/website/api/controllers/deliver-talk-to-us-form-submission.js b/website/api/controllers/deliver-talk-to-us-form-submission.js
index 42abe97909..d07f2474b4 100644
--- a/website/api/controllers/deliver-talk-to-us-form-submission.js
+++ b/website/api/controllers/deliver-talk-to-us-form-submission.js
@@ -73,6 +73,15 @@ module.exports = {
if(_.includes(sails.config.custom.bannedEmailDomainsForWebsiteSubmissions, emailDomain.toLowerCase())){
throw 'invalidEmailDomain';
}
+
+ await sails.helpers.salesforce.updateOrCreateContactAndAccount.with({
+ emailAddress,
+ firstName,
+ lastName,
+ organization: organization,
+ primaryBuyingSituation: primaryBuyingSituation === 'eo-security' ? 'Endpoint operations - Security' : primaryBuyingSituation === 'eo-it' ? 'Endpoint operations - IT' : primaryBuyingSituation === 'mdm' ? 'Device management (MDM)' : 'Vulnerability management',
+ });
+
await sails.helpers.http.post.with({
url: 'https://hooks.zapier.com/hooks/catch/3627242/3cxwxdo/',
data: {
diff --git a/website/api/controllers/entrance/signup.js b/website/api/controllers/entrance/signup.js
index c5f36303f5..e20c3252e1 100644
--- a/website/api/controllers/entrance/signup.js
+++ b/website/api/controllers/entrance/signup.js
@@ -138,6 +138,14 @@ the account verification message.)`,
.intercept({name: 'UsageError'}, 'invalid')
.fetch();
+
+ await sails.helpers.salesforce.updateOrCreateContactAndAccount.with({
+ emailAddress: newEmailAddress,
+ firstName: firstName,
+ lastName: lastName,
+ organization: organization,
+ });
+
// Send a POST request to Zapier
await sails.helpers.http.post.with({
url: 'https://hooks.zapier.com/hooks/catch/3627242/30bq2ib/',
diff --git a/website/api/controllers/get-human-interpretation-from-osquery-sql.js b/website/api/controllers/get-human-interpretation-from-osquery-sql.js
new file mode 100644
index 0000000000..a7d2e18b4a
--- /dev/null
+++ b/website/api/controllers/get-human-interpretation-from-osquery-sql.js
@@ -0,0 +1,106 @@
+module.exports = {
+
+
+ friendlyName: 'Get human interpretation from osquery sql',
+
+
+ description: 'Infer policy information from osquery SQL.',
+
+
+ inputs: {
+
+ sql: {
+ type: 'string',
+ required: true
+ },
+
+ },
+
+
+ exits: {
+
+ success: {
+ outputFriendlyName: 'Humanesque interpretation',
+ outputDescription: 'If the call to the LLM fails, then a success response is sent with an explanation about the failure (e.g. "under heavy load", etc)',
+ outputExample: {
+ risks: 'Using an outdated macOS version risks exposure to security vulnerabilities and potential system instability.',
+ whatWillProbablyHappenDuringMaintenance: 'We will update your macOS to version 14.4.1 to enhance security and stability.'
+ }
+ },
+
+ },
+
+
+ fn: async function ({sql}) {
+
+ if (!sails.config.custom.openAiSecret) {
+ throw new Error('sails.config.custom.openAiSecret not set.');
+ }//•
+
+ // Build our prompt
+ let prompt = `Given this osquery policy: aka a query which either passes (≥1 row) or fails (0 rows) for a given laptop, what risks might we anticipate from that laptop having failed the policy?
+
+Here is the query:
+\`\`\`
+${sql}
+\`\`\`
+
+Remember to minimize the number of words used!
+
+Please give me all of the above in JSON, with this data shape:
+
+{
+ risks: 'TODO',
+ whatWillProbablyHappenDuringMaintenance: 'TODO'
+}
+
+Please do not add any text outside of the JSON report or wrap it in a code fence.`;
+ // Fallback message in case LLM API request fails.
+ let failureMessage = 'Failed to generate human interpretation using generative AI.';
+
+ let BASE_MODEL = 'gpt-4';// The base model to use. https://platform.openai.com/docs/models/gpt-4
+ // (Max tokens for gpt-3.5 ≈≈ 4000) (Max tokens for gpt-4 ≈≈ 8000)
+ // [?] API: https://platform.openai.com/docs/api-reference/chat/create
+ let openAiResponse = await sails.helpers.http.post('https://api.openai.com/v1/chat/completions', {
+ model: BASE_MODEL,
+ messages: [// https://platform.openai.com/docs/guides/chat/introduction
+ {
+ role: 'user',
+ content: prompt
+ }
+ ],
+ temperature: 0.7,
+ max_tokens: 256//eslint-disable-line camelcase
+ }, {
+ Authorization: `Bearer ${sails.config.custom.openAiSecret}`
+ })
+ .tolerate((err)=>{
+ sails.log.warn(failureMessage+' Error details from LLM: '+err.stack);
+ return {
+ choices: [
+ {
+ message: {
+ content: `{ "risks": "${failureMessage}", "whatWillProbablyHappenDuringMaintenance": "${failureMessage}" }`
+ }
+ }
+ ]
+ };
+ });
+
+ let report;
+ try {
+ report = JSON.parse(openAiResponse.choices[0].message.content);
+ } catch (err) {
+ sails.log.warn('When trying to parse a JSON report returned from the Open AI API, an error occurred. Error details from JSON.parse: '+err.stack+'\n Report returned from Open AI:'+openAiResponse.choices[0].message.content);
+ report = {
+ risks: failureMessage,
+ whatWillProbablyHappenDuringMaintenance: failureMessage
+ };
+ }
+
+ return report;
+
+ }
+
+
+};
diff --git a/website/api/controllers/save-questionnaire-progress.js b/website/api/controllers/save-questionnaire-progress.js
index f89b1b116e..36794f96f6 100644
--- a/website/api/controllers/save-questionnaire-progress.js
+++ b/website/api/controllers/save-questionnaire-progress.js
@@ -63,22 +63,6 @@ module.exports = {
.set({
primaryBuyingSituation: primaryBuyingSituation
});
- // Send a POST request to Zapier
- await sails.helpers.http.post.with({
- url: 'https://hooks.zapier.com/hooks/catch/3627242/3pl7yt1/',
- data: {
- primaryBuyingSituation,
- emailAddress: this.req.me.emailAddress,
- webhookSecret: sails.config.custom.zapierSandboxWebhookSecret,
- }
- })
- .timeout(5000)
- .tolerate(['non200Response', 'requestFailed'], (err)=>{
- // Note that Zapier responds with a 2xx status code even if something goes wrong, so just because this message is not logged doesn't mean everything is hunky dory. More info: https://github.com/fleetdm/fleet/pull/6380#issuecomment-1204395762
- sails.log.warn(`When a user completed the 'What are you using Fleet for' questionnaire step, a lead/contact could not be updated in the CRM for this email address: ${this.req.me.emailAddress}. Raw error: ${err}`);
- return;
- });
-
// Set the primary buying situation in the user's session.
this.req.session.primaryBuyingSituation = primaryBuyingSituation;
}//fi
@@ -96,30 +80,32 @@ module.exports = {
// - yes-deployed-local: » Stage 3 (Tried Fleet but might not have a use case)
// - yes-deployed-long-time: Stage 2 (Tried Fleet long ago but might not fully grasp)
// - no: Stage 2 (Never tried Fleet and might not fully grasp)
- // 'how-many-hosts': No change // TODO (see above -- instead of no change there should be a change)
- // 'will-you-be-self-hosting': No change // TODO (see above -- instead of no change there should maybe be a change, sometimes)
+ // 'how-many-hosts': Stage 6
+ // 'will-you-be-self-hosting': Stage 6
// 'what-are-you-working-on-eo-security'
- // - no-use-case-yet: » No change // TODO (see above -- instead of no change there should maybe be a change, sometimes)
+ // - no-use-case-yet: » Stage 2/3 (depends on answer from 'have-you-ever-used-fleet' step)
// - All other options » Stage 4
// 'what-does-your-team-manage-eo-it'
- // - no-use-case-yet: » No change // TODO (see above -- instead of no change there should maybe be a change, sometimes)
+ // - no-use-case-yet: » Stage 2/3 (depends on answer from 'have-you-ever-used-fleet' step)
// - All other options » Stage 4
// 'what-does-your-team-manage-vm'
- // - no-use-case-yet: » No change // TODO (see above -- instead of no change there should maybe be a change, sometimes)
+ // - no-use-case-yet: » Stage 2/3 (depends on answer from 'have-you-ever-used-fleet' step)
// - All other options » Stage 4
// 'what-do-you-manage-mdm'
- // - no-use-case-yet: » No change // TODO (see above -- instead of no change there should maybe be a change, sometimes)
+ // - no-use-case-yet: » Stage 2/3 (depends on answer from 'have-you-ever-used-fleet' step)
// - All other options » Stage 4
- // 'is-it-any-good': // TODO (see above -- instead of no change there should maybe be a change, sometimes)
+ // 'is-it-any-good': Stage 2/3/4 (depends on answer from 'have-you-ever-used-fleet' & the buying situation specific step)
// 'what-did-you-think'
// - deploy-fleet-in-environment » Stage 5
- // - let-me-think-about-it » // TODO (see above -- instead of no change there should maybe be a change, sometimes)
+ // - let-me-think-about-it » Stage 2
// - host-fleet-for-me » N/A (currently not selectable, but should set the user's psychologicalStage to stage 5)
let psychologicalStage = userRecord.psychologicalStage;
// Get the value of the submitted formData, we do this so we only need to check one variable, instead of (formData.attribute === 'foo');
let valueFromFormData = _.values(formData)[0];
- if(currentStep === 'what-are-you-using-fleet-for') {
+ if(currentStep === 'start') {
+ // There is change when the user completes the start step.
+ } else if(currentStep === 'what-are-you-using-fleet-for') {
psychologicalStage = '2 - Aware';
} else if(currentStep === 'have-you-ever-used-fleet') {
if(['yes-deployed', 'yes-recently-deployed'].includes(valueFromFormData)) {
@@ -132,41 +118,94 @@ module.exports = {
// Otherwise, we'll just assume liu're only aware. Maybe liu don't fully grasp what Fleet can do.
psychologicalStage = '2 - Aware';
}
- } else if(['what-are-you-working-on-eo-security','what-does-your-team-manage-eo-it','what-does-your-team-manage-vm','what-do-you-manage-mdm'].includes(currentStep)){
- if(valueFromFormData === 'no-use-case-yet') {
- // If this user doe not have a use case for Fleet yet, set their psyStage to 3
- psychologicalStage = '3 - Intrigued';
- } else {// Otherwise, they have a use case and will be set to stage 4.
- psychologicalStage = '4 - Has use case';
- }
- } else if(currentStep === 'what-did-you-think') {
- // If the user is ready to deploy Fleet in their work environemnt, then they're ready to get buy-in from their team, so set their psyStage to 5.
- if(valueFromFormData === 'deploy-fleet-in-environment') {
- psychologicalStage = '5 - Personally confident';
- }
- // If the user selects let me think about it, their stage will not change.
+ } else {
+ // If the user submitted any other step, we'll set variables using the answers to the previous questions.
+ // Get the user's selected primaryBuyingSiutation.
+ let currentSelectedBuyingSituation = questionnaireProgress['what-are-you-using-fleet-for'].primaryBuyingSituation;
+ // Get the user's answer to the "Have you ever used Fleet?" question.
+ let hasUsedFleetAnswer = questionnaireProgress['have-you-ever-used-fleet'].fleetUseStatus;
+ if(['what-are-you-working-on-eo-security','what-does-your-team-manage-eo-it','what-does-your-team-manage-vm','what-do-you-manage-mdm'].includes(currentStep)){
+ if(valueFromFormData === 'no-use-case-yet') {
+ // Check the user's answer to the previous question
+ if(hasUsedFleetAnswer === 'yes-deployed-local'){
+ // If they've tried Fleet locally, set their stage to 3.
+ psychologicalStage = '3 - Intrigued';
+ } else {
+ psychologicalStage = '2 - Aware';
+ }
+ } else {// Otherwise, they have a use case and will be set to stage 4.
+ psychologicalStage = '4 - Has use case';
+ }
+ } else if(currentStep === 'is-it-any-good') {
+ if(currentSelectedBuyingSituation === 'mdm') {
+ // Since the mdm use case question is the only buying situation-sepcific question where a use case can't
+ // be selected, we'll check the user's previous answers befroe changing their psyStage
+ if(questionnaireProgress['what-do-you-manage-mdm'].mdmUseCase === 'no-use-case-yet'){
+ // Check the user's answer to the have-you-ever-used-fleet question.
+ if(hasUsedFleetAnswer === 'yes-deployed-local') {
+ // If they've tried Fleet locally, set their stage to 3.
+ psychologicalStage = '3 - Intrigued';
+ } else {
+ psychologicalStage = '2 - Aware';
+ }
+ } else {
+ psychologicalStage = '4 - Has use case';
+ }
+ } else {// For any other selected primary buying situation, since a use case will have been selected, set their psyStage to 4
+ psychologicalStage = '4 - Has use case';
+ // FUTURE: check previous answers for other selected buying situations.
+ }
+ } else if(currentStep === 'what-did-you-think') {
+ // If the user is ready to deploy Fleet in their work environemnt, then they're ready to get buy-in from their team, so set their psyStage to 5.
+ if(valueFromFormData === 'deploy-fleet-in-environment') {
+ psychologicalStage = '5 - Personally confident';
+ } else if(valueFromFormData === 'let-me-think-about-it') {
+ // If the user selects "Let me think about it", their stage change to 2
+ psychologicalStage = '2 - Aware';
+ }
+ // If the user selects "I’d like you to host Fleet for me", the form is not submitted, and they are taken to the /contact page instead. FUTURE: set stage to stage 5.
+ } else if(currentStep === 'how-many-hosts') {
+ // If they have Fleet deployed, they have team buy-in
+ psychologicalStage = '6 - Has team buy-in';
+ } else if(currentStep === 'will-you-be-self-hosting') {
+ // If they have Fleet deployed, they have team buy-in
+ psychologicalStage = '6 - Has team buy-in';
+ }//fi
+
}//fi
- // Send a POST request to Zapier
- await sails.helpers.http.post.with({
- url: 'https://hooks.zapier.com/hooks/catch/3627242/3nltwbg/',
- data: {
+ // Only update CRM records if the user's psychological stage changes.
+ if(currentStep !== userRecord.currentStep){
+ await sails.helpers.salesforce.updateOrCreateContactAndAccount.with({
emailAddress: this.req.me.emailAddress,
firstName: this.req.me.firstName,
lastName: this.req.me.lastName,
- primaryBuyingSituation: primaryBuyingSituation,
+ primaryBuyingSituation: primaryBuyingSituation === 'eo-security' ? 'Endpoint operations - Security' : primaryBuyingSituation === 'eo-it' ? 'Endpoint operations - IT' : primaryBuyingSituation === 'mdm' ? 'Device management (MDM)' : primaryBuyingSituation === 'vm' ? 'Vulnerability management' : undefined,
organization: this.req.me.organization,
psychologicalStage,
- currentStep,
- webhookSecret: sails.config.custom.zapierSandboxWebhookSecret,
- }
- })
- .timeout(5000)
- .tolerate(['non200Response', 'requestFailed'], (err)=>{
- // Note that Zapier responds with a 2xx status code even if something goes wrong, so just because this message is not logged doesn't mean everything is hunky dory. More info: https://github.com/fleetdm/fleet/pull/6380#issuecomment-1204395762
- sails.log.warn(`When a user completed a questionnaire step, a lead/contact could not be updated in the CRM for this email address: ${this.req.me.emailAddress}. Raw error: ${err}`);
- return;
- });
+ });
+ }
+ // TODO: send all other answers to Salesforce (when there are fields for them)
+
+ // await sails.helpers.http.post.with({
+ // url: 'https://hooks.zapier.com/hooks/catch/3627242/3nltwbg/',
+ // data: {
+ // emailAddress: this.req.me.emailAddress,
+ // firstName: this.req.me.firstName,
+ // lastName: this.req.me.lastName,
+ // primaryBuyingSituation: primaryBuyingSituation,
+ // organization: this.req.me.organization,
+ // psychologicalStage,
+ // currentStep,
+ // webhookSecret: sails.config.custom.zapierSandboxWebhookSecret,
+ // }
+ // })
+ // .timeout(5000)
+ // .tolerate(['non200Response', 'requestFailed'], (err)=>{
+ // // Note that Zapier responds with a 2xx status code even if something goes wrong, so just because this message is not logged doesn't mean everything is hunky dory. More info: https://github.com/fleetdm/fleet/pull/6380#issuecomment-1204395762
+ // sails.log.warn(`When a user completed a questionnaire step, a lead/contact could not be updated in the CRM for this email address: ${this.req.me.emailAddress}. Raw error: ${err}`);
+ // return;
+ // });
// Set the user's answer to the current step.
questionnaireProgress[currentStep] = formData;
// Clone the questionnaireProgress to prevent any mutations from sending it through the updateOne Waterline method.
diff --git a/website/api/controllers/view-endpoint-ops.js b/website/api/controllers/view-endpoint-ops.js
index e67c72cdce..ec913c6972 100644
--- a/website/api/controllers/view-endpoint-ops.js
+++ b/website/api/controllers/view-endpoint-ops.js
@@ -23,13 +23,19 @@ module.exports = {
// Get testimonials for the component.
let testimonialsForScrollableTweets = _.clone(sails.config.builtStaticContent.testimonials);
- // Filter the testimonials by product category
- testimonialsForScrollableTweets = _.filter(testimonialsForScrollableTweets, (testimonial)=>{
- return _.contains(testimonial.productCategories, 'Endpoint operations');
- });
// Specify an order for the testimonials on this page using the last names of quote authors
- let testimonialOrderForThisPage = ['Charles Zaffery','Dan Grzelak','Nico Waisman','Tom Larkin','Austin Anderson','Erik Gomez','Nick Fohs','Brendan Shaklovitz','Mike Arpaia','Andre Shields','Dhruv Majumdar','Ahmed Elshaer','Abubakar Yousafzai','Harrison Ravazzolo','Wes Whetstone','Kenny Botelho', 'Chandra Majumdar'];
+ let testimonialOrderForThisPage = ['Charles Zaffery','Dan Grzelak','Nico Waisman','Tom Larkin','Austin Anderson','Erik Gomez','Nick Fohs','Brendan Shaklovitz','Mike Arpaia','Andre Shields','Dhruv Majumdar','Ahmed Elshaer','Abubakar Yousafzai','Harrison Ravazzolo','Wes Whetstone','Kenny Botelho', 'Chandra Majumdar','Eric Tan'];
+ if(['eo-it', 'mdm'].includes(this.req.session.primaryBuyingSituation)){
+ testimonialOrderForThisPage = [ 'Harrison Ravazzolo', 'Eric Tan','Erik Gomez', 'Tom Larkin', 'Nick Fohs', 'Wes Whetstone', 'Mike Arpaia', 'Kenny Botelho'];
+ } else if(['eo-security', 'vm'].includes(this.req.session.primaryBuyingSituation)){
+ testimonialOrderForThisPage = ['Nico Waisman','Charles Zaffery','Abubakar Yousafzai','Eric Tan','Mike Arpaia','Chandra Majumdar','Ahmed Elshaer','Brendan Shaklovitz','Austin Anderson','Dan Grzelak','Dhruv Majumdar'];
+ }
+ // Filter the testimonials by product category and the filtered list we built above.
+ testimonialsForScrollableTweets = _.filter(testimonialsForScrollableTweets, (testimonial)=>{
+ return _.contains(testimonial.productCategories, 'Endpoint operations') && _.contains(testimonialOrderForThisPage, testimonial.quoteAuthorName);
+ });
+
testimonialsForScrollableTweets.sort((a, b)=>{
if(testimonialOrderForThisPage.indexOf(a.quoteAuthorName) === -1){
return 1;
diff --git a/website/api/helpers/iq/get-enriched.js b/website/api/helpers/iq/get-enriched.js
new file mode 100644
index 0000000000..c347069380
--- /dev/null
+++ b/website/api/helpers/iq/get-enriched.js
@@ -0,0 +1,251 @@
+module.exports = {
+
+
+ friendlyName: 'Get enriched',
+
+
+ description: 'Search for the contact indicated and return enriched data.',
+
+
+ extendedDescription: `Note about coresignal.com from their FAQ:
+ Q: Do you have emails or phone numbers in your database?
+ A: No, we don't have emails or phone numbers. We only hold publicly available data on companies and professionals.`,
+
+
+ moreInfoUrl: 'https://coresignal.com/faq/',
+
+
+ inputs: {
+
+ emailAddress: { type: 'string', defaultsTo: '', },
+ linkedinUrl: { type: 'string', defaultsTo: '', },
+ firstName: { type: 'string', defaultsTo: '', },
+ lastName: { type: 'string', defaultsTo: '', },
+ organization: { type: 'string', defaultsTo: '', },
+
+ },
+
+
+ exits: {
+
+ success: {
+ outputFriendlyName: 'Report',
+ outputDescription: 'All available, enriched info about this person and their current employer.',
+ outputType: {
+ person: {
+ linkedinUrl: 'string',
+ firstName: 'string',
+ lastName: 'string',
+ organization: 'string',
+ title: 'string',
+ },
+ employer: {
+ organization: 'string',
+ numberOfEmployees: 'number',
+ emailDomain: 'string',
+ linkedinCompanyPageUrl: 'string',
+ }
+ }
+ },
+
+ },
+
+
+ fn: async function ({emailAddress,linkedinUrl,firstName,lastName,organization}) {
+
+ require('assert')(sails.config.custom.iqSecret);// FUTURE: Rename this config
+
+ let RX_PROTOCOL_AND_COMMON_SUBDOMAINS = /^https?\:\/\/(www\.|about\.)*/;
+
+ sails.log.verbose('Enriching from…', emailAddress,linkedinUrl,firstName,lastName,organization);
+
+ // Gather initial information that is obtainable just from parsing provided inputs.
+ let emailDomain;
+ if (emailAddress) {
+ let matches = emailAddress.match(/@([^@]+)$/);
+ if (Array.isArray(matches)) {
+ emailDomain = matches[1] || undefined;
+ }
+ }//fi
+
+ let linkedinPersonIdOrUrlSlug;
+ if (linkedinUrl) {
+ let matches = linkedinUrl.match(/linkedin\.com\/in\/([^/]+)\/?$/);
+ if (Array.isArray(matches)) {
+ linkedinPersonIdOrUrlSlug = matches[1] || undefined;
+ }
+ }//fi
+
+
+ // If no linkedin URL was provided for the person, then also do a website+name+orgName search
+ // vs contacts to try and locate the person's linkedin URL.
+ //
+ // [?] Why? It provides us with a better unique id than an email. For example, consider
+ // how everyone has more than one email. This way, we can avoid sending any emails that
+ // people might experience as "spam", even if they unsubscribe from a different email.
+ if (!linkedinPersonIdOrUrlSlug && (firstName || lastName || emailAddress)) {
+ let searchBy = {};
+ if (firstName && !lastName) {
+ searchBy.name = firstName;
+ } else if (!firstName && lastName) {
+ searchBy.name = lastName;
+ } else if (firstName && lastName) {
+ searchBy.name = firstName + ' ' + lastName;
+ } else {
+ searchBy.name = _.startCase(emailAddress.replace(/@[^@]+$/,'').replace(/\./g,' ').replace(/[0-9\-]/g,''));
+ }
+ if (emailDomain) {
+ searchBy.experience_company_website_url = emailDomain;//eslint-disable-line camelcase
+ searchBy.active_experience = true;//eslint-disable-line camelcase
+ }//fi
+ if (organization) {
+ searchBy.experience_company_name = organization;//eslint-disable-line camelcase
+ searchBy.active_experience = true;//eslint-disable-line camelcase
+ }//fi
+ if (Object.keys(searchBy).length >= 1) {
+ // [?] https://dashboard.coresignal.com/get-started
+ let matchingLinkedinPersonIds = await sails.helpers.http.post('https://api.coresignal.com/cdapi/v1/linkedin/member/search/filter', searchBy, {
+ Authorization: `Bearer ${sails.config.custom.iqSecret}`,
+ 'content-type': 'application/json'
+ }).tolerate((err)=>{
+ sails.log.warn(`Failed to enrich (${emailAddress},${linkedinUrl},${firstName},${lastName},${organization}):`,err);
+ return [];
+ });
+ linkedinPersonIdOrUrlSlug = matchingLinkedinPersonIds[0];
+ }//fi
+ }//fi
+
+ let person;
+ let matchingLinkedinCompanyPageId;
+
+ if (linkedinPersonIdOrUrlSlug) {
+ // [?] https://dashboard.coresignal.com/get-started
+ let matchingPersonInfo = await sails.helpers.http.get('https://api.coresignal.com/cdapi/v1/linkedin/member/collect/'+encodeURIComponent(linkedinPersonIdOrUrlSlug), {}, {
+ Authorization: `Bearer ${sails.config.custom.iqSecret}`,
+ 'content-type': 'application/json'
+ }).tolerate((err)=>{
+ sails.log.warn(`Failed to enrich (${emailAddress},${linkedinUrl},${firstName},${lastName},${organization}):`,err);
+ return undefined;
+ });
+
+ if (matchingPersonInfo) {
+
+ require('assert')(Array.isArray(matchingPersonInfo.member_experience_collection));
+ let matchingWorkExperience = (
+ matchingPersonInfo.member_experience_collection.filter((workExperience) =>
+ !workExperience.deleted &&
+ workExperience.order_in_profile === 1 &&
+ !workExperience.date_to
+ // FUTURE: Be smarter by also trying to match the stated organization, if one is provided, for the edge case where someone has multiple current positions.
+ )
+ )[0];
+
+ let matchedOrganizationName;
+ let matchedTitle;
+ if (matchingWorkExperience) {
+ matchedOrganizationName = matchingWorkExperience.company_name;
+ matchedTitle = matchingWorkExperience.title;
+ matchingLinkedinCompanyPageId = matchingWorkExperience.company_id;// « save for use below
+ }
+
+ person = {
+ linkedinUrl: matchingPersonInfo.canonical_url.replace(RX_PROTOCOL_AND_COMMON_SUBDOMAINS,''),
+ firstName: matchingPersonInfo.first_name,
+ lastName: matchingPersonInfo.last_name,
+ organization: matchedOrganizationName || '',
+ title: matchedTitle || ''
+ };
+
+ if (linkedinUrl && person.linkedinUrl && person.linkedinUrl !== linkedinUrl) {
+ sails.log.warn(`Unexpected result when enriching: Matched linkedin URL for person (${person.linkedinUrl}) does not equal the provided linkedin URL (${linkedinUrl})`);
+ }//fi
+ if (firstName && person.firstName && person.firstName !== firstName) {
+ sails.log.warn(`Unexpected result when enriching: Matched current firstName for person (${person.firstName}) does not equal the provided "firstName" (${firstName})`);
+ }//fi
+ if (lastName && person.lastName && person.lastName !== lastName) {
+ sails.log.warn(`Unexpected result when enriching: Matched current lastName for person (${person.lastName}) does not equal the provided "lastName" (${lastName})`);
+ }//fi
+ if (organization && person.organization && person.organization !== organization) {
+ sails.log.warn(`Unexpected result when enriching: Matched current TOP organization for person (${person.organization}) does not equal the provided "organization" (${organization})`);
+ }//fi
+ }//fi
+ }//fi
+
+
+
+
+ // Now look up the employer.
+ //
+ // [?] Either use the matched linkedin company page ID from above,
+ // or if no match, then try to find the linkedin company page ID
+ // by other means. If nothing works, then give up and don't enrich.
+ if (!matchingLinkedinCompanyPageId) {
+ let searchBy = {};
+ if (emailDomain) {
+ searchBy.website = emailDomain;
+ }//fi
+ if (organization) {
+ searchBy.name = organization;
+ }//fi
+ if (Object.keys(searchBy).length >= 1) {
+ // [?] https://dashboard.coresignal.com/get-started
+ let matchingLinkedinCompanyPageIds = await sails.helpers.http.post('https://api.coresignal.com/cdapi/v1/linkedin/company/search/filter', searchBy, {
+ Authorization: `Bearer ${sails.config.custom.iqSecret}`,
+ 'content-type': 'application/json'
+ }).tolerate((err)=>{
+ sails.log.warn(`Failed to enrich (${emailAddress},${linkedinUrl},${firstName},${lastName},${organization}):`,err);
+ return [];
+ });
+
+ // If name and domain were used for searching the org, yet no matches found,
+ // try searching again, but this time w/o the org name.
+ if (matchingLinkedinCompanyPageIds.length === 0 && searchBy.name && searchBy.website) {
+ delete searchBy.name;
+ // [?] https://dashboard.coresignal.com/get-started
+ matchingLinkedinCompanyPageIds = await sails.helpers.http.post('https://api.coresignal.com/cdapi/v1/linkedin/company/search/filter', searchBy, {
+ Authorization: `Bearer ${sails.config.custom.iqSecret}`,
+ 'content-type': 'application/json'
+ }).tolerate((err)=>{
+ sails.log.warn(`Failed to enrich (${emailAddress},${linkedinUrl},${firstName},${lastName},${organization}):`,err);
+ return [];
+ });
+ }//fi
+
+ matchingLinkedinCompanyPageId = matchingLinkedinCompanyPageIds[0];
+ }//fi
+ }//fi
+
+ let employer;
+ if (matchingLinkedinCompanyPageId) {
+ // [?] https://dashboard.coresignal.com/get-started
+ let matchingCompanyPageInfo = await sails.helpers.http.get('https://api.coresignal.com/cdapi/v1/linkedin/company/collect/'+encodeURIComponent(matchingLinkedinCompanyPageId), {}, {
+ Authorization: `Bearer ${sails.config.custom.iqSecret}`,
+ 'content-type': 'application/json'
+ }).tolerate((err)=>{
+ sails.log.warn(`Failed to enrich (${emailAddress},${linkedinUrl},${firstName},${lastName},${organization}):`,err);
+ return undefined;
+ });
+ if (matchingCompanyPageInfo) {
+ employer = {
+ organization: matchingCompanyPageInfo.name,
+ numberOfEmployees: matchingCompanyPageInfo.employees_count,
+ emailDomain: require('url').parse(matchingCompanyPageInfo.website).hostname.replace(RX_PROTOCOL_AND_COMMON_SUBDOMAINS,''),
+ linkedinCompanyPageUrl: matchingCompanyPageInfo.canonical_url.replace(RX_PROTOCOL_AND_COMMON_SUBDOMAINS,''),
+ };
+ if (organization && employer.organization && employer.organization !== organization) {
+ sails.log.warn(`Unexpected result when enriching: Matched organization name (${employer.organization}) does not equal the provided "organization" (${organization})`);
+ }//fi
+ if (emailDomain && employer.emailDomain && employer.emailDomain !== emailDomain) {
+ sails.log.warn(`Unexpected result when enriching: Email domain inferred from matched organization website (${employer.emailDomain}) does not equal the parsed email domain (${emailDomain}) that was derived from the provided "emailAddress" (${emailAddress})`);
+ }//fi
+ }//fi
+ }//fi
+
+ return {
+ person,
+ employer
+ };
+
+ }
+
+};
diff --git a/website/api/helpers/salesforce/create-lead.js b/website/api/helpers/salesforce/create-lead.js
new file mode 100644
index 0000000000..8ae92c399a
--- /dev/null
+++ b/website/api/helpers/salesforce/create-lead.js
@@ -0,0 +1,90 @@
+module.exports = {
+
+
+ friendlyName: 'Create lead',// FUTURE: Retire this in favor of createTask()
+
+
+ description: 'Create a Lead record in Salesforce representing some kind of action Fleet needs to take for someone, whether based on a signal from their behavior or their explicit request.',
+
+
+ inputs: {
+
+ salesforceAccountId: { type: 'string', required: true },
+ salesforceContactId: { type: 'string', required: true },
+ leadDescription: { type: 'string', description: 'A description of what this lead is about; e.g. a contact form message, or the size of t-shirt being requested.' },
+ leadSource: { type: 'string', required: true, isIn: ['Website - Contact forms', 'Website - Sign up', 'Website - Waitlist', 'Website - swag request'], },// TODO verify and complete enum
+
+
+ // FUTURE: Move these off eventually:
+ firstName: { type: 'string', required: true, description: 'The first name of the referenced contact.' },
+ lastName: { type: 'string', required: true, description: 'The last name of the referenced contact.' },
+ emailAddress: { type: 'string', description: 'The email address of the referenced contact.', extendedDescription: 'Included here so that the little Salesforce thingie that shows email and calendar activity shows maximum contact in both the Contact and Lead views.' },
+ primaryBuyingSituation: { type: 'string' },
+ numberOfHosts: { type: 'number' },
+
+ },
+
+
+ exits: {
+
+ success: {
+ extendedDescription: 'Note that this deliberately has no return value.',
+ },
+
+ },
+
+
+ fn: async function ({salesforceAccountId, salesforceContactId, leadDescription, leadSource, firstName, lastName, emailAddress, primaryBuyingSituation, numberOfHosts}) {
+ require('assert')(sails.config.custom.salesforceIntegrationUsername);
+ require('assert')(sails.config.custom.salesforceIntegrationPasskey);
+ let jsforce = require('jsforce');
+ console.log(firstName, lastName, emailAddress, primaryBuyingSituation, numberOfHosts);
+ let salesforceConnection = new jsforce.Connection({
+ loginUrl : 'https://fleetdm.my.salesforce.com'
+ });
+ await salesforceConnection.login(sails.config.custom.salesforceIntegrationUsername, sails.config.custom.salesforceIntegrationPasskey);
+ // Get the contact record
+ let contactRecord = await salesforceConnection.sobject('Contact')
+ .retrieve(salesforceContactId);
+ // Verify that the account ID provided is valid.
+ let accountRecord = await salesforceConnection.sobject('Account')
+ .retrieve(salesforceAccountId);
+
+ // TODO better error messages
+ if(contactRecord === null) {
+ throw new Error(`When attempting to create a Salesforce lead using the ID of a Contact record, no Contact matching the id provided (${salesforceContactId} was found.`);
+ }
+ if(accountRecord === null) {
+ throw new Error(`When attempting to create a Salesforce lead, no account matching the id provided (${salesforceContactId} could be found`);
+ }
+
+ // TODO: wrap this in a try-catch block to handle errors from Salesforce.
+ // Create the new Lead record.
+ let lead = await salesforceConnection.sobject('Lead')
+ .create({
+ FirstName: contactRecord.FirstName,
+ LastName: contactRecord.LastName,
+ Email: contactRecord.Email,
+ Website: contactRecord.Website,
+ // eslint-disable-next-line camelcase
+ of_hosts__c: contactRecord.of_hosts__c,
+ // eslint-disable-next-line camelcase
+ Primary_buying_scenario__c: contactRecord.Primary_buying_situation__c,
+ // eslint-disable-next-line camelcase
+ LinkedIn_profile__c: contactRecord.LinkedIn_profile__c,
+ Description: leadDescription,
+ LeadSource: leadSource,
+ // eslint-disable-next-line camelcase
+ Contact_associated_by_website__c: salesforceContactId,
+ // eslint-disable-next-line camelcase
+ Account__c: salesforceAccountId,
+ OwnerId: accountRecord.OwnerId
+ });
+ console.log(`Created lead! ${lead}`);
+
+ // TODO handle duplicate leads:
+ }
+
+
+};
+
diff --git a/website/api/helpers/salesforce/create-task.js b/website/api/helpers/salesforce/create-task.js
new file mode 100644
index 0000000000..f509094db2
--- /dev/null
+++ b/website/api/helpers/salesforce/create-task.js
@@ -0,0 +1,45 @@
+module.exports = {
+
+ // TODO: Change this into "create activity" instead (and create github issues for tasks instead)
+ friendlyName: 'Create task',
+
+
+ description: 'Create a task for our team related to a particular account in Salesforce.',
+
+
+ inputs: {
+ salesforceAccountId: { type: 'string', required: true, extendedDescription: 'This account will be used to determine the assignee (owner) for this new task. (The account\'s owner will also be the owner for the new task.)' },
+ dueDate: { type: 'string', example: 'YYYY-MM-DD', extendedDescription: 'If unspecified, defaults to the current date.', regex: /^[0-9][0-9][0-9][0-9]\-[0-9][0-9]\-[0-9][0-9]$/ },
+ },
+
+
+ exits: {
+
+ success: {
+ extendedDescription: 'Note that this deliberately has no return value.',
+ },
+
+ },
+
+
+ fn: async function ({ salesforceAccountId, dueDate }) {
+ sails.log(salesforceAccountId, dueDate);
+ throw new Error('Not yet implemented');
+
+ // require('assert')(sails.config.custom.salesforceSecret);
+
+ // let jsforce = require('jsforce');
+ // let conn = new jsforce.Connection({ /* */ });
+ // let userInfo = await conn.login('___________', '____________');// TODO
+ // let salesforceIntegrationUserId = userInfo.userId;
+
+ // await conn.sobject('Task').create({
+ // DueDate: dueDate,
+ // ActivityDate: jsforce.Date.TODAY
+ // });
+
+ }
+
+
+};
+
diff --git a/website/api/helpers/salesforce/update-or-create-contact-and-account.js b/website/api/helpers/salesforce/update-or-create-contact-and-account.js
new file mode 100644
index 0000000000..8ad95413ea
--- /dev/null
+++ b/website/api/helpers/salesforce/update-or-create-contact-and-account.js
@@ -0,0 +1,210 @@
+module.exports = {
+
+
+ friendlyName: 'Update or create contact and account',
+
+
+ description: 'Upsert contact±account into Salesforce given fresh data about a particular person, and fresh IQ-enrichment data about the person and account.',
+
+
+ inputs: {
+
+ // Find by…
+ emailAddress: { type: 'string' },
+ linkedinUrl: { type: 'string' },
+
+ // Set…
+ firstName: { type: 'string', required: true },
+ lastName: { type: 'string', required: true },
+ organization: { type: 'string' },
+ primaryBuyingSituation: { type: 'string' },
+ psychologicalStage: {
+ type: 'string',
+ isIn: [
+ '1 - Unaware',
+ '2 - Aware',
+ '3 - Intrigued',
+ '4 - Has use case',
+ '5 - Personally confident',
+ '6 - Has team buy-in'
+ ]
+ },
+ },
+
+
+ exits: {
+
+ success: {
+ outputType: {
+ salesforceAccountId: 'string',
+ salesforceContactId: 'string'
+ }
+ },
+
+ },
+
+
+ fn: async function ({emailAddress, linkedinUrl, firstName, lastName, organization, primaryBuyingSituation, psychologicalStage}) {
+ if(sails.config.environment !== 'production') {
+ sails.log.verbose('Skipping Salesforce integration...');
+ return;
+ }
+
+ require('assert')(sails.config.custom.salesforceIntegrationUsername);
+ require('assert')(sails.config.custom.salesforceIntegrationPasskey);
+ require('assert')(sails.config.custom.iqSecret);
+
+
+ if(!emailAddress && !linkedinUrl){
+ throw new Error('UsageError: when updating or creating a contact and account in salesforce, either an email or linkedInUrl is required.');
+ }
+ // Send the information we have to the enrichment helper.
+ let enrichmentData = await sails.helpers.iq.getEnriched(emailAddress, linkedinUrl, firstName, lastName, organization);
+ // console.log(enrichmentData);
+
+ // Log in to Salesforce.
+ let jsforce = require('jsforce');
+ let salesforceConnection = new jsforce.Connection({
+ loginUrl : 'https://fleetdm.my.salesforce.com'
+ });
+
+ let salesforceAccountOwnerId;
+
+ await salesforceConnection.login(sails.config.custom.salesforceIntegrationUsername, sails.config.custom.salesforceIntegrationPasskey);
+
+ let salesforceAccountId;
+ if(!enrichmentData.employer || !enrichmentData.employer.emailDomain || !enrichmentData.employer.organization) {
+ // Special sacraficial meat cave where the contacts with no organization go.
+ // https://fleetdm.lightning.force.com/lightning/r/Account/0014x000025JC8DAAW/view
+ salesforceAccountId = '0014x000025JC8DAAW';
+ salesforceAccountOwnerId = '0054x00000735wDAAQ';
+ } else {
+ let existingAccountRecord = await salesforceConnection.sobject('Account')
+ .findOne({
+ 'Website': enrichmentData.employer.emailDomain,
+ // 'LinkedIn_company_URL__c': enrichmentData.employer.linkedinCompanyPageUrl // TODO: if this information is not present on an existing account, nothing will be returned.
+ });
+ // console.log(existingAccountRecord);
+ if(existingAccountRecord) {
+ // Store the ID of the Account record we found.
+ salesforceAccountId = existingAccountRecord.Id;
+ salesforceAccountOwnerId = existingAccountRecord.OwnerId;
+ // console.log('exising account found!', salesforceAccountId);
+ } else {
+
+
+ let roundRobinUsers = await salesforceConnection.sobject('User')
+ .find({
+ AE_Round_robin__c: true,// eslint-disable-line camelcase
+ });
+ let userWithEarliestAssignTimeStamp = _.sortBy(roundRobinUsers, 'AE_Account_Assignment_round_robin__c')[0];
+
+ let today = new Date();
+ let nowOn = today.toISOString().replace('Z', '+0000');
+
+ salesforceAccountOwnerId = userWithEarliestAssignTimeStamp.Id;
+
+ // Update this user to putthem atthe bottom of the round robin list.
+ await salesforceConnection.sobject('User')
+ .update({
+ Id: salesforceAccountOwnerId,
+ // eslint-disable-next-line camelcase
+ AE_Account_Assignment_round_robin__c: nowOn
+ });
+ // If no existing account record was found, create a new one.
+ let newAccountRecord = await salesforceConnection.sobject('Account')
+ .create({
+ OwnerId: salesforceAccountOwnerId,
+ Account_Assigned_date__c: nowOn,// eslint-disable-line camelcase
+ // eslint-disable-next-line camelcase
+ Current_Assignment_Reason__c: 'Inbound Lead',// TODO verify that this matters. if not, do not set it.
+ Prospect_Status__c: 'Assigned',// eslint-disable-line camelcase
+
+ Name: enrichmentData.employer.organization,// IFWMIH: We know organization exists
+ Website: enrichmentData.employer.emailDomain,
+ LinkedIn_company_URL__c: enrichmentData.employer.linkedinCompanyPageUrl,// eslint-disable-line camelcase
+ NumberOfEmployees: enrichmentData.employer.numberOfEmployees,
+ });
+ salesforceAccountId = newAccountRecord.id;
+ // console.log('New account created!', salesforceAccountId);
+ }
+ }
+
+
+
+ // Now search for an existing Contact.
+ // FUTURE: expand this section to improve the searches.
+ let existingContactRecord;
+ if(emailAddress){
+ // console.log('searching for existing contact by emailAddress');
+ existingContactRecord = await salesforceConnection.sobject('Contact')
+ .findOne({
+ AccountId: salesforceAccountId,
+ Email: emailAddress,
+ });
+ } else if(linkedinUrl) {
+ // console.log('searching for existing contact by linkedInUrl');
+ existingContactRecord = await salesforceConnection.sobject('Contact')
+ .findOne({
+ AccountId: salesforceAccountId,
+ LinkedIn_profile__c: linkedinUrl // eslint-disable-line camelcase
+ });
+ } else {
+ existingContactRecord = undefined;
+ }
+
+ let salesforceContactId;
+ let valuesToSet = {};
+ if(emailAddress){
+ valuesToSet.Email = emailAddress;
+ }
+ if(linkedinUrl || (enrichmentData.person && enrichmentData.person.linkedinUrl)){
+ valuesToSet.LinkedIn_profile__c = linkedinUrl || enrichmentData.person.linkedinUrl;// eslint-disable-line camelcase
+ }
+ if(enrichmentData.person && enrichmentData.person.title){
+ valuesToSet.Title = enrichmentData.person.title;
+ }
+ if(primaryBuyingSituation) {
+ valuesToSet.Primary_buying_situation__c = primaryBuyingSituation;// eslint-disable-line camelcase
+ }
+ if(psychologicalStage) {
+ valuesToSet.Stage__c = psychologicalStage;// eslint-disable-line camelcase
+ }
+
+
+ if(existingContactRecord){
+ salesforceContactId = existingContactRecord.Id;
+ // console.log(`existing contact record found! ${salesforceContactId}`);
+ // Update the existing contact with the information provided.
+ await salesforceConnection.sobject('Contact')
+ .update({
+ Id: salesforceContactId,
+ ...valuesToSet,
+ });
+ // console.log(`${salesforceContactId} updated!`);
+ } else {
+ // Otherwise create a new Contact record.
+ let newContactRecord = await salesforceConnection.sobject('Contact')
+ .create({
+ AccountId: salesforceAccountId,
+ OwnerId: salesforceAccountOwnerId,
+ FirstName: firstName,
+ LastName: lastName,
+ ...valuesToSet,
+ });
+ // console.log(newContactRecord);
+ salesforceContactId = newContactRecord.id;
+ // console.log(`New contact record created! ${salesforceContactId}`);
+ }
+
+
+ return {
+ salesforceAccountId,
+ salesforceContactId
+ };
+
+ }
+
+
+};
+
diff --git a/website/assets/images/articles/enhancing-fleets-vulnerability-management-with-vulncheck-integration-1600x900@2x.png b/website/assets/images/articles/enhancing-fleets-vulnerability-management-with-vulncheck-integration-1600x900@2x.png
new file mode 100644
index 0000000000..fad1e5a4b0
Binary files /dev/null and b/website/assets/images/articles/enhancing-fleets-vulnerability-management-with-vulncheck-integration-1600x900@2x.png differ
diff --git a/website/assets/images/articles/fleet-4.49.0-1600x900@2x.png b/website/assets/images/articles/fleet-4.49.0-1600x900@2x.png
new file mode 100644
index 0000000000..1f85b9c2b9
Binary files /dev/null and b/website/assets/images/articles/fleet-4.49.0-1600x900@2x.png differ
diff --git a/website/assets/js/cloud.setup.js b/website/assets/js/cloud.setup.js
index cf4a555bd5..ab9ebae327 100644
--- a/website/assets/js/cloud.setup.js
+++ b/website/assets/js/cloud.setup.js
@@ -13,7 +13,7 @@
Cloud.setup({
/* eslint-disable */
- methods: {"downloadSitemap":{"verb":"GET","url":"/sitemap.xml","args":[]},"downloadRssFeed":{"verb":"GET","url":"/rss/:categoryName","args":["categoryName"]},"receiveUsageAnalytics":{"verb":"POST","url":"/api/v1/webhooks/receive-usage-analytics","args":["anonymousIdentifier","fleetVersion","licenseTier","numHostsEnrolled","numUsers","numTeams","numPolicies","numLabels","softwareInventoryEnabled","vulnDetectionEnabled","systemUsersEnabled","hostsStatusWebHookEnabled","numWeeklyActiveUsers","numWeeklyPolicyViolationDaysActual","numWeeklyPolicyViolationDaysPossible","hostsEnrolledByOperatingSystem","hostsEnrolledByOrbitVersion","hostsEnrolledByOsqueryVersion","storedErrors","numHostsNotResponding","organization","mdmMacOsEnabled","mdmWindowsEnabled","liveQueryDisabled","hostExpiryEnabled"]},"receiveFromGithub":{"verb":"GET","url":"/api/v1/webhooks/github","args":["botSignature","action","sender","repository","changes","issue","comment","pull_request","label","release"]},"receiveFromStripe":{"verb":"POST","url":"/api/v1/webhooks/receive-from-stripe","args":["id","type","data","webhookSecret"]},"deliverContactFormMessage":{"verb":"POST","url":"/api/v1/deliver-contact-form-message","args":["emailAddress","firstName","lastName","message"]},"sendPasswordRecoveryEmail":{"verb":"POST","url":"/api/v1/entrance/send-password-recovery-email","args":["emailAddress"]},"signup":{"verb":"POST","url":"/api/v1/customers/signup","args":["emailAddress","password","organization","firstName","lastName","signupReason","primaryBuyingSituation"]},"updateProfile":{"verb":"POST","url":"/api/v1/account/update-profile","args":["firstName","lastName","organization","emailAddress"]},"updatePassword":{"verb":"POST","url":"/api/v1/account/update-password","args":["oldPassword","newPassword"]},"updateBillingCard":{"verb":"POST","url":"/api/v1/account/update-billing-card","args":["stripeToken","billingCardLast4","billingCardBrand","billingCardExpMonth","billingCardExpYear"]},"login":{"verb":"POST","url":"/api/v1/customers/login","args":["emailAddress","password","rememberMe"]},"logout":{"verb":"GET","url":"/api/v1/account/logout","args":[]},"createQuote":{"verb":"POST","url":"/api/v1/customers/create-quote","args":["numberOfHosts"]},"saveBillingInfoAndSubscribe":{"verb":"POST","url":"/api/v1/customers/save-billing-info-and-subscribe","args":["quoteId","organization","firstName","lastName","paymentSource"]},"updatePasswordAndLogin":{"verb":"POST","url":"/api/v1/entrance/update-password-and-login","args":["password","token"]},"deliverDemoSignup":{"verb":"POST","url":"/api/v1/deliver-demo-signup","args":["emailAddress"]},"createOrUpdateOneNewsletterSubscription":{"verb":"POST","url":"/api/v1/create-or-update-one-newsletter-subscription","args":["emailAddress","subscribeTo"]},"unsubscribeFromAllNewsletters":{"verb":"GET","url":"/api/v1/unsubscribe-from-all-newsletters","args":["emailAddress"]},"buildLicenseKey":{"verb":"POST","url":"/api/v1/admin/build-license-key","args":["numberOfHosts","organization","expiresAt","partnerName"]},"createVantaAuthorizationRequest":{"verb":"POST","url":"/api/v1/create-vanta-authorization-request","args":["emailAddress","fleetInstanceUrl","fleetApiKey"]},"deliverMdmBetaSignup":{"verb":"POST","url":"/api/v1/deliver-mdm-beta-signup","args":["emailAddress","fullName","jobTitle","numberOfHosts"]},"deliverAppleCsr":{"verb":"POST","url":"/api/v1/deliver-apple-csr","args":["unsignedCsrData"]},"deliverLaunchPartySignup":{"verb":"POST","url":"/api/v1/deliver-launch-party-signup","args":["emailAddress","firstName","lastName","jobTitle","phoneNumber"]},"deliverMdmDemoEmail":{"verb":"POST","url":"/api/v1/deliver-mdm-demo-email","args":["emailAddress"]},"provisionSandboxInstanceAndDeliverEmail":{"verb":"POST","url":"/api/v1/admin/provision-sandbox-instance-and-deliver-email","args":["userId"]},"deliverTalkToUsFormSubmission":{"verb":"POST","url":"/api/v1/deliver-talk-to-us-form-submission","args":["emailAddress","firstName","lastName","organization","numberOfHosts","primaryBuyingSituation"]},"saveQuestionnaireProgress":{"verb":"POST","url":"/api/v1/save-questionnaire-progress","args":["currentStep","formData"]}}
+ methods: {"downloadSitemap":{"verb":"GET","url":"/sitemap.xml","args":[]},"downloadRssFeed":{"verb":"GET","url":"/rss/:categoryName","args":["categoryName"]},"receiveUsageAnalytics":{"verb":"POST","url":"/api/v1/webhooks/receive-usage-analytics","args":["anonymousIdentifier","fleetVersion","licenseTier","numHostsEnrolled","numUsers","numTeams","numPolicies","numLabels","softwareInventoryEnabled","vulnDetectionEnabled","systemUsersEnabled","hostsStatusWebHookEnabled","numWeeklyActiveUsers","numWeeklyPolicyViolationDaysActual","numWeeklyPolicyViolationDaysPossible","hostsEnrolledByOperatingSystem","hostsEnrolledByOrbitVersion","hostsEnrolledByOsqueryVersion","storedErrors","numHostsNotResponding","organization","mdmMacOsEnabled","mdmWindowsEnabled","liveQueryDisabled","hostExpiryEnabled"]},"receiveFromGithub":{"verb":"GET","url":"/api/v1/webhooks/github","args":["botSignature","action","sender","repository","changes","issue","comment","pull_request","label","release"]},"receiveFromStripe":{"verb":"POST","url":"/api/v1/webhooks/receive-from-stripe","args":["id","type","data","webhookSecret"]},"deliverContactFormMessage":{"verb":"POST","url":"/api/v1/deliver-contact-form-message","args":["emailAddress","firstName","lastName","message"]},"sendPasswordRecoveryEmail":{"verb":"POST","url":"/api/v1/entrance/send-password-recovery-email","args":["emailAddress"]},"signup":{"verb":"POST","url":"/api/v1/customers/signup","args":["emailAddress","password","organization","firstName","lastName","signupReason"]},"updateProfile":{"verb":"POST","url":"/api/v1/account/update-profile","args":["firstName","lastName","organization","emailAddress"]},"updatePassword":{"verb":"POST","url":"/api/v1/account/update-password","args":["oldPassword","newPassword"]},"updateBillingCard":{"verb":"POST","url":"/api/v1/account/update-billing-card","args":["stripeToken","billingCardLast4","billingCardBrand","billingCardExpMonth","billingCardExpYear"]},"login":{"verb":"POST","url":"/api/v1/customers/login","args":["emailAddress","password","rememberMe"]},"logout":{"verb":"GET","url":"/api/v1/account/logout","args":[]},"createQuote":{"verb":"POST","url":"/api/v1/customers/create-quote","args":["numberOfHosts"]},"saveBillingInfoAndSubscribe":{"verb":"POST","url":"/api/v1/customers/save-billing-info-and-subscribe","args":["quoteId","organization","firstName","lastName","paymentSource"]},"updatePasswordAndLogin":{"verb":"POST","url":"/api/v1/entrance/update-password-and-login","args":["password","token"]},"deliverDemoSignup":{"verb":"POST","url":"/api/v1/deliver-demo-signup","args":["emailAddress"]},"createOrUpdateOneNewsletterSubscription":{"verb":"POST","url":"/api/v1/create-or-update-one-newsletter-subscription","args":["emailAddress","subscribeTo"]},"unsubscribeFromAllNewsletters":{"verb":"GET","url":"/api/v1/unsubscribe-from-all-newsletters","args":["emailAddress"]},"buildLicenseKey":{"verb":"POST","url":"/api/v1/admin/build-license-key","args":["numberOfHosts","organization","expiresAt","partnerName"]},"createVantaAuthorizationRequest":{"verb":"POST","url":"/api/v1/create-vanta-authorization-request","args":["emailAddress","fleetInstanceUrl","fleetApiKey"]},"deliverMdmBetaSignup":{"verb":"POST","url":"/api/v1/deliver-mdm-beta-signup","args":["emailAddress","fullName","jobTitle","numberOfHosts"]},"getHumanInterpretationFromOsquerySql":{"verb":"POST","url":"/api/v1/get-human-interpretation-from-osquery-sql","args":["sql"]},"deliverAppleCsr":{"verb":"POST","url":"/api/v1/deliver-apple-csr","args":["unsignedCsrData"]},"deliverMdmDemoEmail":{"verb":"POST","url":"/api/v1/deliver-mdm-demo-email","args":["emailAddress"]},"provisionSandboxInstanceAndDeliverEmail":{"verb":"POST","url":"/api/v1/admin/provision-sandbox-instance-and-deliver-email","args":["userId"]},"deliverTalkToUsFormSubmission":{"verb":"POST","url":"/api/v1/deliver-talk-to-us-form-submission","args":["emailAddress","firstName","lastName","organization","numberOfHosts","primaryBuyingSituation"]},"saveQuestionnaireProgress":{"verb":"POST","url":"/api/v1/save-questionnaire-progress","args":["currentStep","formData"]}}
/* eslint-enable */
});
diff --git a/website/assets/styles/pages/endpoint-ops.less b/website/assets/styles/pages/endpoint-ops.less
index fb766e9cd2..410f45c88b 100644
--- a/website/assets/styles/pages/endpoint-ops.less
+++ b/website/assets/styles/pages/endpoint-ops.less
@@ -242,14 +242,14 @@
&:hover {
box-shadow: 0px 4px 16px 0px #E2E4EA;
}
- &:first-of-type {
+ &.austin-anderson {
background: url('/images/video-testimonial-thumbnail-austin-anderson-223x168@2x.jpg');
background-position: center;
background-size: cover;
margin-right: 12px;
margin-left: 0px;
}
- &:last-of-type {
+ &.nick-fohs {
background: url('/images/video-testimonial-thumbnail-nick-fohs-223x168@2x.png');
background-position: center;
background-size: cover;
diff --git a/website/assets/styles/pages/homepage.less b/website/assets/styles/pages/homepage.less
index 249fdf7441..455e28d27d 100644
--- a/website/assets/styles/pages/homepage.less
+++ b/website/assets/styles/pages/homepage.less
@@ -132,7 +132,7 @@
&:hover {
box-shadow: 0px 4px 16px 0px #E2E4EA;
}
- &:first-of-type {
+ &.nick-fohs {
[purpose='testimonial-video'] {
background: url('/images/video-testimonial-thumbnail-nick-fohs-160x120@2x.png');
background-position: center;
@@ -141,7 +141,7 @@
margin-right: 20px;
margin-left: 0px;
}
- &:last-of-type {
+ &.austin-anderson {
[purpose='testimonial-video'] {
background: url('/images/video-testimonial-thumbnail-austin-anderson-160x120@2x.png');
background-position: center;
@@ -730,22 +730,27 @@
[purpose='category-text-block'] {
max-width: 410px;
}
+ [purpose='platform-block'] {
+ .left {
+ margin-right: 24px;
+ }
+ .right {
+ margin-left: 24px;
+ }
+ }
[purpose='endpoint-ops-image'] {
- margin-right: 24px;
img {
height: auto;
width: 100%;
}
}
[purpose='device-management-image'] {
- margin-left: 24px;
img {
height: auto;
width: 100%;
}
}
[purpose='vuln-management-image'] {
- margin-left: 24px;
img {
height: auto;
width: 100%;
@@ -935,9 +940,18 @@
width: 100%;
}
}
+ [purpose='platform-block'] {
+ .left {
+ margin-right: auto;
+ margin-left: auto;
+ }
+ .right {
+ margin-right: auto;
+ margin-left: auto;
+ }
+ }
[purpose='endpoint-ops-image'] {
- margin-right: auto;
- margin-left: auto;
+
margin-bottom: 20px;
img {
width: 100%;
diff --git a/website/config/custom.js b/website/config/custom.js
index 1823de7a3a..c46e9c634b 100644
--- a/website/config/custom.js
+++ b/website/config/custom.js
@@ -85,6 +85,13 @@ module.exports.custom = {
// || (Or if you don't need billing, feel free to remove them.)
//--------------------------------------------------------------------------
+
+ // Other integrations:
+ // openAiSecret: undefined,
+ // iqSecret: undefined, // You gotta use the base64-encoded API secret. (Get it in your account settings in LeadIQ.)
+ // salesforceIntegrationUsername: undefined,
+ // salesforceIntegrationPasskey: undefined,
+
// ██████╗ ██████╗ ██╗███████╗
// ██╔══██╗██╔══██╗██║██╔════╝
// ██║ ██║██████╔╝██║███████╗
@@ -148,7 +155,7 @@ module.exports.custom = {
// 'website/views/pages/pricing.ejs': '', // « Covered in CODEOWNERS (2023-07-22)
// 'handbook/company/pricing-features-table.yml': '', // « Covered in CODEOWNERS (2023-07-22)
- '/handbook/company/testimonials.yml': 'mike-j-thomas',
+ 'handbook/company/testimonials.yml': 'mike-j-thomas',
// 🫧 Other brandfronts
'README.md': 'mikermcneil',// « GitHub brandfront
@@ -251,7 +258,7 @@ module.exports.custom = {
'handbook/sales': ['sampfluger88','mikermcneil'],
'handbook/demand': ['sampfluger88','mikermcneil'],
'handbook/customer-success': ['sampfluger88','mikermcneil'],
- '/handbook/company/testimonials.yml': ['eashaw', 'mike-j-thomas', 'sampfluger88', 'mikermcneil'],
+ 'handbook/company/testimonials.yml': ['eashaw', 'mike-j-thomas', 'sampfluger88', 'mikermcneil'],
// GitHub issue templates
'.github/ISSUE_TEMPLATE': ['mikermcneil', 'lukeheath', 'sampfluger88'],
@@ -286,10 +293,52 @@ module.exports.custom = {
'/': ['lukeheath'] // Future update this
},
+ // ███████╗ ██████╗██╗ ██╗███████╗███╗ ███╗ █████╗
+ // ██╔════╝██╔════╝██║ ██║██╔════╝████╗ ████║██╔══██╗
+ // ███████╗██║ ███████║█████╗ ██╔████╔██║███████║
+ // ╚════██║██║ ██╔══██║██╔══╝ ██║╚██╔╝██║██╔══██║
+ // ███████║╚██████╗██║ ██║███████╗██║ ╚═╝ ██║██║ ██║
+ // ╚══════╝ ╚═════╝╚═╝ ╚═╝╚══════╝╚═╝ ╚═╝╚═╝ ╚═╝
+ //
// The version of osquery to use when generating schema docs
// (both in Fleet's query console and on fleetdm.com)
versionOfOsquerySchemaToUseWhenGeneratingDocumentation: '5.11.0',
+ // ███████╗██╗ ██╗██████╗ ██╗ ██████╗ ██████╗ ███████╗ ██████╗ █████╗ ████████╗ █████╗
+ // ██╔════╝╚██╗██╔╝██╔══██╗██║ ██╔═══██╗██╔══██╗██╔════╝ ██╔══██╗██╔══██╗╚══██╔══╝██╔══██╗
+ // █████╗ ╚███╔╝ ██████╔╝██║ ██║ ██║██████╔╝█████╗ ██║ ██║███████║ ██║ ███████║
+ // ██╔══╝ ██╔██╗ ██╔═══╝ ██║ ██║ ██║██╔══██╗██╔══╝ ██║ ██║██╔══██║ ██║ ██╔══██║
+ // ███████╗██╔╝ ██╗██║ ███████╗╚██████╔╝██║ ██║███████╗ ██████╔╝██║ ██║ ██║ ██║ ██║
+ // ╚══════╝╚═╝ ╚═╝╚═╝ ╚══════╝ ╚═════╝ ╚═╝ ╚═╝╚══════╝ ╚═════╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝
+ //
+ // Config variables in this section are used for the /try-fleet/explore-data page on fleetdm.com
+
+ // For sending requests to a Fleet instance:
+ // fleetBaseUrlForQueryReports: '…',
+ // fleetTokenForQueryReports: '…',
+
+ // The API ID of the team of hosts created for query reports.
+ // teamApidForQueryReports:
+
+ // A dictionary where each key is the name of an osquery table, and the value is the API ID of the query that selects all information from that table. e.g., {'account_policy_data': 2045, 'ad_config': 2047, …}
+ // queryIdsByTableName: {…}
+
+ // A dictionary where each key is the lowercased platform, and the value is the API ID of a host. e.g., {'macos': 92, 'windows': 94, 'linux': 93}
+ // hostIdsByHostPlatform: {…}
+
+ // ███╗ ███╗██╗███████╗ ██████╗
+ // ████╗ ████║██║██╔════╝██╔════╝
+ // ██╔████╔██║██║███████╗██║
+ // ██║╚██╔╝██║██║╚════██║██║
+ // ██║ ╚═╝ ██║██║███████║╚██████╗
+ // ╚═╝ ╚═╝╚═╝╚══════╝ ╚═════╝
+ //
+ /***************************************************************************
+ * *
+ * Any other custom config this Sails app should use during development. *
+ * (and possibly in ALL environments, if not overridden in config/env/) *
+ * *
+ ***************************************************************************/
// FUTURE: Consolidate these two lists of email domains (And maybe find another word for banned)
// For the deliver-apple-csr webhook:
@@ -339,34 +388,6 @@ module.exports.custom = {
'ymail.com',
],
- // ███████╗██╗ ██╗██████╗ ██╗ ██████╗ ██████╗ ███████╗ ██████╗ █████╗ ████████╗ █████╗
- // ██╔════╝╚██╗██╔╝██╔══██╗██║ ██╔═══██╗██╔══██╗██╔════╝ ██╔══██╗██╔══██╗╚══██╔══╝██╔══██╗
- // █████╗ ╚███╔╝ ██████╔╝██║ ██║ ██║██████╔╝█████╗ ██║ ██║███████║ ██║ ███████║
- // ██╔══╝ ██╔██╗ ██╔═══╝ ██║ ██║ ██║██╔══██╗██╔══╝ ██║ ██║██╔══██║ ██║ ██╔══██║
- // ███████╗██╔╝ ██╗██║ ███████╗╚██████╔╝██║ ██║███████╗ ██████╔╝██║ ██║ ██║ ██║ ██║
- // ╚══════╝╚═╝ ╚═╝╚═╝ ╚══════╝ ╚═════╝ ╚═╝ ╚═╝╚══════╝ ╚═════╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝
- //
- // Config variables in this section are used for the /try-fleet/explore-data page on fleetdm.com
-
- // For sending requests to a Fleet instance:
- // fleetBaseUrlForQueryReports: '…',
- // fleetTokenForQueryReports: '…',
-
- // The API ID of the team of hosts created for query reports.
- // teamApidForQueryReports:
-
- // A dictionary where each key is the name of an osquery table, and the value is the API ID of the query that selects all information from that table. e.g., {'account_policy_data': 2045, 'ad_config': 2047, …}
- // queryIdsByTableName: {…}
-
- // A dictionary where each key is the lowercased platform, and the value is the API ID of a host. e.g., {'macos': 92, 'windows': 94, 'linux': 93}
- // hostIdsByHostPlatform: {…}
-
- /***************************************************************************
- * *
- * Any other custom config this Sails app should use during development. *
- * (and possibly in ALL environments, if not overridden in config/env/) *
- * *
- ***************************************************************************/
// Contact form:
// slackWebhookUrlForContactForm: '…',
diff --git a/website/config/policies.js b/website/config/policies.js
index ebecbbea74..2c46a7f477 100644
--- a/website/config/policies.js
+++ b/website/config/policies.js
@@ -53,5 +53,6 @@ module.exports.policies = {
'try-fleet/view-explore-data': true,
'try-fleet/view-query-report': true,
'deliver-talk-to-us-form-submission': true,
+ 'get-human-interpretation-from-osquery-sql': true,
'customers/view-new-license': true,
};
diff --git a/website/config/routes.js b/website/config/routes.js
index daa4c7b4bc..7ad2a790a0 100644
--- a/website/config/routes.js
+++ b/website/config/routes.js
@@ -492,6 +492,7 @@ module.exports.routes = {
'GET /learn-more-about/enabling-calendar-api': 'https://console.cloud.google.com/apis/library/calendar-json.googleapis.com',
'GET /learn-more-about/downgrading': '/docs/using-fleet/downgrading-fleet',
'GET /learn-more-about/fleetd': '/docs/get-started/anatomy#fleetd',
+ 'GET /learn-more-about/rotating-enroll-secrets': '/docs/configuration/configuration-files#rotating-enroll-secrets',
// Sitemap
// =============================================================================================================
@@ -552,6 +553,7 @@ module.exports.routes = {
'POST /api/v1/admin/build-license-key': { action: 'admin/build-license-key' },
'POST /api/v1/create-vanta-authorization-request': { action: 'create-vanta-authorization-request' },
'POST /api/v1/deliver-mdm-beta-signup': { action: 'deliver-mdm-beta-signup' },
+ 'POST /api/v1/get-human-interpretation-from-osquery-sql': { action: 'get-human-interpretation-from-osquery-sql', csrf: false },
'POST /api/v1/deliver-apple-csr ': { action: 'deliver-apple-csr', csrf: false},
'POST /api/v1/deliver-mdm-demo-email': { action: 'deliver-mdm-demo-email' },
'POST /api/v1/admin/provision-sandbox-instance-and-deliver-email': { action: 'admin/provision-sandbox-instance-and-deliver-email' },
diff --git a/website/package.json b/website/package.json
index 987acade2e..a5f0902560 100644
--- a/website/package.json
+++ b/website/package.json
@@ -8,6 +8,7 @@
"@sailshq/connect-redis": "^6.1.3",
"@sailshq/lodash": "^3.10.5",
"@sailshq/socket.io-redis": "^6.1.2",
+ "jsforce": "1.11.1",
"jsonwebtoken": "9.0.2",
"moment": "2.29.4",
"sails": "^1.5.10",
diff --git a/website/views/layouts/layout-email.ejs b/website/views/layouts/layout-email.ejs
index d79fb6aa6d..1895bcca1c 100644
--- a/website/views/layouts/layout-email.ejs
+++ b/website/views/layouts/layout-email.ejs
@@ -13,7 +13,7 @@
%>)