From ffc5c5a91ea0ddad863833ec9ec07745663a69b8 Mon Sep 17 00:00:00 2001 From: Zach Wasserman Date: Mon, 3 May 2021 08:10:27 -0700 Subject: [PATCH 01/51] Add package-lock.json to .gitignore (#708) --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index fe0513dee2..20f75d9f03 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,7 @@ cypress/downloads # Fleet local development DB backups backup.sql.gz + +# Common mistake for new developers to run npm install and then end up +# committing a package-lock.json. Fleet app uses Yarn with yarn.lock. +package-lock.json From b8aec634139e4ef2ca900c50ee1265034d017542 Mon Sep 17 00:00:00 2001 From: RachelElysia <71795832+RachelElysia@users.noreply.github.com> Date: Mon, 3 May 2021 18:58:23 -0400 Subject: [PATCH 02/51] Host details page styling (#710) * Host details page restyled to match Figma * Page uses flexboxes * Text match Figma --- assets/images/icon-chevron-down-9x6@2x.png | Bin 0 -> 408 bytes .../hosts/HostDetailsPage/HostDetailsPage.jsx | 55 ++++++++++-------- .../pages/hosts/HostDetailsPage/_styles.scss | 50 ++++++++++++---- 3 files changed, 71 insertions(+), 34 deletions(-) create mode 100644 assets/images/icon-chevron-down-9x6@2x.png diff --git a/assets/images/icon-chevron-down-9x6@2x.png b/assets/images/icon-chevron-down-9x6@2x.png new file mode 100644 index 0000000000000000000000000000000000000000..46c0151dfe773ed186915a143068ba7b8b261fa7 GIT binary patch literal 408 zcmV;J0cZY+P)HK~#7F?Uv6` zgHRC0cLVB`QqqDQmRlm>e;?!2<#Dw)`C}7N=j*JSPTP@wB~MRVL1*CI)1QB3=`n48o-} z-^xIb_&;3UlMv%iFuTS(5#sI>%gbyY<|M?Z0=?p(m9BV=b_Mo{Zy|5IT|()NNrGto z-H2bf64x@Y`N5KzhAzcx#65s&zB@q&TSRQ&Y|SHpK9z7Y^OMPDNQP*?!}O}inR%>) zwYJ|b+GF_g7Q_HPlyFmt+bPDRiP0XYO;la+tb`+!$aPH^uj1@ANQm*M#&u0todwxE zNPA5bcp@Rzr(wrLyBnnW4u43BluEv{T}&R&!TAFkjF}8CSsNk%0000 query an
offline host. - @@ -219,7 +222,7 @@ export class HostDetailsPage extends Component { }); return ( -
+

Labels

{labels.length === 0 ? (

No labels are associated with this host.

@@ -344,6 +347,12 @@ export class HostDetailsPage extends Component { Object.keys(object).forEach((key) => { if (object[key] === "") { object[key] = "--"; + } else if ( + key === "logger_tls_period" || + key === "config_tls_refresh" || + key === "distributed_interval" + ) { + object[key] = `${object[key]} sec`; } }); }); @@ -356,6 +365,12 @@ export class HostDetailsPage extends Component { return (
+
+ + back chevron + Back to Hosts + +
@@ -395,22 +410,20 @@ export class HostDetailsPage extends Component {
{renderActionButtons()}
-
+

About this host

- Last seen - Enrolled - Uptime -
-
- - {humanHostLastSeen(aboutData.seen_time)} - + Created at {humanHostEnrolled(aboutData.last_enrolled_at)} + Updated at + + {humanHostLastSeen(aboutData.seen_time)} + + Uptime {humanHostUptime(aboutData.uptime)} @@ -419,33 +432,27 @@ export class HostDetailsPage extends Component {
Hardware model - Serial number - IP address -
-
{aboutData.hardware_model} + Serial number {aboutData.hardware_serial} + IPv4 {aboutData.primary_ip}
-
-

Osquery configuration

-
-
- Config refresh +
+

Agent Options

+
+
+ Config TLS refresh {osqueryData.config_tls_refresh} -
-
Logger TLS period {osqueryData.logger_tls_period} -
-
Distributed interval {osqueryData.distributed_interval} diff --git a/frontend/pages/hosts/HostDetailsPage/_styles.scss b/frontend/pages/hosts/HostDetailsPage/_styles.scss index 126722c77e..f56ed3adba 100644 --- a/frontend/pages/hosts/HostDetailsPage/_styles.scss +++ b/frontend/pages/hosts/HostDetailsPage/_styles.scss @@ -1,17 +1,34 @@ +a { + font-size: $x-small; + color: $core-vibrant-blue; + font-weight: $bold; + text-decoration: none; +} + .host-details { display: flex; - flex-direction: column; + flex-wrap: wrap; + flex-grow: 1; + align-content: flex-start; padding-bottom: 50px; min-width: 0; + background-color: $ui-off-white; + gap: $pad-large; .section { + flex: 100%; display: flex; flex-direction: column; - margin: $pad-xxlarge 0 0; + background-color: $core-white; + border-radius: 16px; + border: 1px solid $ui-fleet-blue-15; + padding: $pad-xxlarge; + box-shadow: 0px 3px 0px rgba(226, 228, 234, 0.4); &__header { - margin: 0 0 $pad-medium; font-size: $medium; + font-weight: bold; + margin: 0 0 $pad-medium 0; } .info { @@ -94,7 +111,8 @@ } } - .about { + .about, + .osquery { .info { &__item { &--about { @@ -107,18 +125,25 @@ display: flex; flex-direction: column; margin-right: $pad-xxlarge; + } + &__data { + margin-bottom: $pad-medium; - span { - margin-bottom: $pad-medium; - - &:last-child { - margin-bottom: 0; - } + &:last-child { + margin-bottom: 0; } } } } + .col-50 { + flex: 2; + } + + .col-25 { + flex: 1; + } + .status { color: $core-fleet-black; text-transform: capitalize; @@ -224,4 +249,9 @@ white-space: nowrap; } } + + #back-chevron { + width: 12px; + margin-right: 10px; + } } From dbe0cdeb446377c67d67911872ba52b687f8fe8d Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Mon, 3 May 2021 19:53:04 -0500 Subject: [PATCH 03/51] call for contributors --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 37e044d4d0..ced3e6e7be 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,10 @@ Documentation for Fleet can be found [here on GitHub](./docs/README.md). Please join us in the #fleet channel on [osquery Slack](https://osquery.slack.com/join/shared_invite/zt-h29zm0gk-s2DBtGUTW4CFel0f0IjTEw#/). +#### Contributing + +Contributions are welcome, whether you answer questions on Slack/GitHub/StackOverflow/Twitter, improve the documentation or website, write a tutorial, give a talk, start a local osquery meetup, troubleshoot reported issues, or [submit a patch](https://github.com/fleetdm/fleet/blob/master/CONTRIBUTING.md). The Fleet code of conduct is [on GitHub](https://github.com/fleetdm/fleet/blob/master/CODE_OF_CONDUCT.md). + #### Community projects Below are some projects created by Fleet community members. Please submit a pull request if you'd like your project featured. From 3b81cb88e9581aae97225fa14f85ab1b30e110da Mon Sep 17 00:00:00 2001 From: Zach Wasserman Date: Tue, 4 May 2021 11:09:23 -0700 Subject: [PATCH 04/51] Update documentation on size of Fleet deployments (#718) Better represents the growth of Fleet deployment size. --- docs/1-Using-Fleet/FAQ.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/1-Using-Fleet/FAQ.md b/docs/1-Using-Fleet/FAQ.md index 35c6ffed14..bfc6db9efb 100644 --- a/docs/1-Using-Fleet/FAQ.md +++ b/docs/1-Using-Fleet/FAQ.md @@ -1,4 +1,5 @@ # Using Fleet FAQ + - [Has anyone stress tested Fleet? How many clients can the Fleet server handle?](#has-anyone-stress-tested-fleet-how-many-clients-can-the-fleet-server-handle) - [How often do labels refresh? Is the refresh frequency configurable?](#how-often-do-labels-refresh-is-the-refresh-frequency-configurable) - [How do I revoke the authorization tokens for a user?](#how-do-i-revoke-the-authorization-tokens-for-a-user) @@ -10,9 +11,9 @@ ## Has anyone stress tested Fleet? How many clients can the Fleet server handle? -Fleet has been stress tested to 150,000 online hosts and 400,000 total enrolled hosts. There are numerous production deployments in the thousands, in the tens of thousands of hosts range, and there are production deployments in the high tens of thousands of hosts range. +Fleet has been stress tested to 150,000 online hosts and 400,000 total enrolled hosts. Production deployments exist with over 100,000 hosts and numerous production deployments manage tens of thousands of hosts. -It’s standard deployment practice to have multiple Fleet servers behind a load balancer. However, typically the MySQL database is the bottleneck and an individual Fleet server can handle tens of thousands of hosts. +It’s standard deployment practice to have multiple Fleet servers behind a load balancer. However, typically the MySQL database is the performance bottleneck and a single Fleet server can handle tens of thousands of hosts. ## How often do labels refresh? Is the refresh frequency configurable? @@ -30,7 +31,6 @@ Fleet can live query the `osquery_schedule` table. Performing this live query al Fleet provides standard interfaces for monitoring and alerting. See the [Monitoring Fleet](./5-Monitoring-Fleet.md) documentation for details. - ## Why is the “Add User” button disabled? The “Add User” button is disabled if SMTP (email) has not been configured for the Fleet server. Currently, there is no way to add new users without email capabilities. @@ -51,7 +51,7 @@ It is possible to configure osqueryd to log query results outside of Fleet. For ### What are my options for storing the osquery logs? -Folks typically use Fleet to ship logs to data aggregation systems like Splunk, the ELK stack, and Graylog. +Folks typically use Fleet to ship logs to data aggregation systems like Splunk, the ELK stack, and Graylog. The [logger configuration options](https://github.com/fleetdm/fleet/blob/master/docs/2-Deployment/2-Configuration.md#osquery_status_log_plugin) allow you to select the log output plugin. Using the log outputs you can route the logs to your chosen aggregation system. @@ -67,4 +67,4 @@ Expecting results, but not seeing anything in the logs? ## Why aren’t my live queries being logged? -Live query results are never logged to the filesystem of the Fleet server. See [Where are my query results?](#where-are-my-query-results). \ No newline at end of file +Live query results are never logged to the filesystem of the Fleet server. See [Where are my query results?](#where-are-my-query-results). From 9fb9da31f5462fa7dda4819a114bbdbc0252c347 Mon Sep 17 00:00:00 2001 From: noahtalerman <47070608+noahtalerman@users.noreply.github.com> Date: Tue, 4 May 2021 15:50:18 -0400 Subject: [PATCH 05/51] Bring Orbit docs into Fleet docs and add FAQ section (#717) - Add new `2-Orbit-osquery/` directory to the top-level `docs/` directory. - Rename `2-Deployment/` -> `3-Deployment/` to accommodate new Orbit directory. - Rename `3-Contribution/` -> `4-Contribution/` to accommodate new Orbit directory. - Add FAQ section to Orbit documentation. --- CHANGELOG.md | 2 +- docs/1-Using-Fleet/2-fleetctl-CLI.md | 6 +- docs/1-Using-Fleet/3-REST-API.md | 2 +- docs/1-Using-Fleet/4-Osquery-logs.md | 14 +- .../6-Security-best-practices.md | 2 +- docs/1-Using-Fleet/7-Updating-Fleet.md | 2 +- docs/1-Using-Fleet/FAQ.md | 8 +- docs/2-Orbit-osquery/README.md | 152 ++++++++++++++++++ .../1-Installation.md | 0 .../2-Configuration.md | 0 .../3-Adding-hosts.md | 0 .../4-Example-deployment-scenarios.md | 0 .../4-fleetctl-agent-updates.md | 0 docs/{2-Deployment => 3-Deployment}/FAQ.md | 0 docs/{2-Deployment => 3-Deployment}/README.md | 0 .../1-Building-Fleet.md | 0 .../2-Testing.md | 0 .../3-Migrations.md | 0 .../4-Committing-Changes.md | 0 .../5-Releasing-Fleet.md | 0 .../{3-Contribution => 4-Contribution}/FAQ.md | 0 .../README.md | 0 docs/README.md | 11 +- frontend/README.md | 2 +- .../hosts/AddHostModal/AddHostModal.jsx | 2 +- tools/backup_db/README.md | 2 +- website/config/routes.js | 2 +- 27 files changed, 181 insertions(+), 26 deletions(-) create mode 100644 docs/2-Orbit-osquery/README.md rename docs/{2-Deployment => 3-Deployment}/1-Installation.md (100%) rename docs/{2-Deployment => 3-Deployment}/2-Configuration.md (100%) rename docs/{2-Deployment => 3-Deployment}/3-Adding-hosts.md (100%) rename docs/{2-Deployment => 3-Deployment}/4-Example-deployment-scenarios.md (100%) rename docs/{2-Deployment => 3-Deployment}/4-fleetctl-agent-updates.md (100%) rename docs/{2-Deployment => 3-Deployment}/FAQ.md (100%) rename docs/{2-Deployment => 3-Deployment}/README.md (100%) rename docs/{3-Contribution => 4-Contribution}/1-Building-Fleet.md (100%) rename docs/{3-Contribution => 4-Contribution}/2-Testing.md (100%) rename docs/{3-Contribution => 4-Contribution}/3-Migrations.md (100%) rename docs/{3-Contribution => 4-Contribution}/4-Committing-Changes.md (100%) rename docs/{3-Contribution => 4-Contribution}/5-Releasing-Fleet.md (100%) rename docs/{3-Contribution => 4-Contribution}/FAQ.md (100%) rename docs/{3-Contribution => 4-Contribution}/README.md (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 241644d9f7..e156fd7e7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ * Improve Fleet performance by batch updating host seen time instead of updating synchronously. This improvement reduces MySQL CPU usage by ~33% with 4,000 simulated hosts and MySQL running in Docker. -* Add support for software inventory, introducing a list of installed software items on each host's respective _Host details_ page. This feature is flagged off by default (for now). Check out [the feature flag documentation for instructions on how to turn this feature on](https://github.com/fleetdm/fleet/blob/master/docs/2-Deployment/2-Configuration.md#software-inventory). +* Add support for software inventory, introducing a list of installed software items on each host's respective _Host details_ page. This feature is flagged off by default (for now). Check out [the feature flag documentation for instructions on how to turn this feature on](https://github.com/fleetdm/fleet/blob/master/docs/3-Deployment/2-Configuration.md#software-inventory). * Add Windows support for `fleetctl` agent autoupdates. The `fleetctl updates` command provides the ability to self-manage an agent update server. Available for Fleet Basic customers. diff --git a/docs/1-Using-Fleet/2-fleetctl-CLI.md b/docs/1-Using-Fleet/2-fleetctl-CLI.md index 755f1c218f..cf85636d6c 100644 --- a/docs/1-Using-Fleet/2-fleetctl-CLI.md +++ b/docs/1-Using-Fleet/2-fleetctl-CLI.md @@ -34,7 +34,7 @@ This guide illustrates: ### Running Fleet -For the sake of this tutorial, I will be using the local development Docker Compose infrastructure to run Fleet locally. This is documented in some detail in the [developer documentation](../3-Contribution/1-Building-Fleet.md#development-infrastructure), but the following are the minimal set of commands that you can run from the root of the repository (assuming that you have a working Go/JavaScript toolchain installed along with Docker Compose): +For the sake of this tutorial, I will be using the local development Docker Compose infrastructure to run Fleet locally. This is documented in some detail in the [developer documentation](../4-Contribution/1-Building-Fleet.md#development-infrastructure), but the following are the minimal set of commands that you can run from the root of the repository (assuming that you have a working Go/JavaScript toolchain installed along with Docker Compose): ``` docker-compose up -d @@ -80,7 +80,7 @@ It's possible to specify the password via the `--password` flag or the `$PASSWOR ### Connecting a host -For the sake of this tutorial, I'm going to be using Kolide's osquery launcher to start osquery locally and connect it to Fleet. To learn more about connecting osquery to Fleet, see the [Adding Hosts to Fleet](../2-Deployment/3-Adding-hosts.md) documentation. +For the sake of this tutorial, I'm going to be using Kolide's osquery launcher to start osquery locally and connect it to Fleet. To learn more about connecting osquery to Fleet, see the [Adding Hosts to Fleet](../3-Deployment/3-Adding-hosts.md) documentation. To get your osquery enroll secret, run the following: @@ -577,7 +577,7 @@ spec: Fleet supports osquery's file carving functionality as of Fleet 3.3.0. This allows the Fleet server to request files (and sets of files) from osquery agents, returning the full contents to Fleet. -File carving data can be either stored in Fleet's database or to an external S3 bucket. For information on how to configure the latter, consult the [configuration docs](https://github.com/fleetdm/fleet/blob/master/docs/2-Deployment/2-Configuration.md#s3-file-carving-backend). +File carving data can be either stored in Fleet's database or to an external S3 bucket. For information on how to configure the latter, consult the [configuration docs](https://github.com/fleetdm/fleet/blob/master/docs/3-Deployment/2-Configuration.md#s3-file-carving-backend). ### Configuration diff --git a/docs/1-Using-Fleet/3-REST-API.md b/docs/1-Using-Fleet/3-REST-API.md index 7822b9aba7..f4962fe1cd 100644 --- a/docs/1-Using-Fleet/3-REST-API.md +++ b/docs/1-Using-Fleet/3-REST-API.md @@ -589,7 +589,7 @@ None. Returns the information of the specified host. -The endpoint returns the host's installed `software` if the software inventory feature flag is turned on. This feature flag is turned off by default. [Check out the feature flag documentation](../2-Deployment/2-Configuration.md#feature-flags) for instructions on how to turn on the software inventory feature. +The endpoint returns the host's installed `software` if the software inventory feature flag is turned on. This feature flag is turned off by default. [Check out the feature flag documentation](../3-Deployment/2-Configuration.md#feature-flags) for instructions on how to turn on the software inventory feature. `GET /api/v1/fleet/hosts/{id}` diff --git a/docs/1-Using-Fleet/4-Osquery-logs.md b/docs/1-Using-Fleet/4-Osquery-logs.md index d51b300fef..2550b1a4ee 100644 --- a/docs/1-Using-Fleet/4-Osquery-logs.md +++ b/docs/1-Using-Fleet/4-Osquery-logs.md @@ -22,21 +22,21 @@ Fleet supports the following logging plugins for osquery logs: - [PubSub](#pubsub) - Logs are written to Google Cloud PubSub topics. - [Stdout](#stdout) - Logs are written to stdout. -To set the osquery logging plugins, use the `--osquery_result_log_plugin` and `--osquery_status_log_plugin` flags (or [equivalents for environment variables or configuration files](../2-Deployment/2-Configuration.md#options)). +To set the osquery logging plugins, use the `--osquery_result_log_plugin` and `--osquery_status_log_plugin` flags (or [equivalents for environment variables or configuration files](../3-Deployment/2-Configuration.md#options)). ### Filesystem The default logging plugin. - Plugin name: `filesystem` -- Flag namespace: [filesystem](../2-Deployment/2-Configuration.md#filesystem) +- Flag namespace: [filesystem](../3-Deployment/2-Configuration.md#filesystem) With the filesystem plugin, osquery result and/or status logs are written to the local filesystem on the Fleet server. This is typically used with a log forwarding agent on the Fleet server that will push the logs into a logging pipeline. Note that if multiple load-balanced Fleet servers are used, the logs will be load-balanced across those servers (not duplicated). ### Firehose - Plugin name: `firehose` -- Flag namespace: [firehose](../2-Deployment/2-Configuration.md#firehose) +- Flag namespace: [firehose](../3-Deployment/2-Configuration.md#firehose) With the Firehose plugin, osquery result and/or status logs are written to [AWS Firehose](https://aws.amazon.com/kinesis/data-firehose/) streams. This is a very good method for aggregating osquery logs into AWS S3 storage. @@ -45,7 +45,7 @@ Note that Firehose logging has limits [discussed in the documentation](https://d ### Kinesis - Plugin name: `kinesis` -- Flag namespace: [kinesis](../2-Deployment/2-Configuration.md#kinesis) +- Flag namespace: [kinesis](../3-Deployment/2-Configuration.md#kinesis) With the Kinesis plugin, osquery result and/or status logs are written to [AWS Kinesis](https://aws.amazon.com/kinesis/data-streams) streams. @@ -58,7 +58,7 @@ output in the Fleet logs and those logs _will not_ be sent to Kinesis. ### Lambda - Plugin name: `lambda` -- Flag namespace: [lambda](../2-Deployment/2-Configuration.md#lambda) +- Flag namespace: [lambda](../3-Deployment/2-Configuration.md#lambda) With the Lambda plugin, osquery result and/or status logs are written to [AWS Lambda](https://aws.amazon.com/lambda/) functions. @@ -73,7 +73,7 @@ output in the Fleet logs and those logs _will not_ be sent to Lambda. ### PubSub - Plugin name: `pubsub` -- Flag namespace: [pubsub](../2-Deployment/2-Configuration.md#pubsub) +- Flag namespace: [pubsub](../3-Deployment/2-Configuration.md#pubsub) With the PubSub plugin, osquery result and/or status logs are written to [PubSub](https://cloud.google.com/pubsub/) topics. @@ -82,7 +82,7 @@ Note that messages over 10MB will be dropped, with a notification sent to the fl ### Stdout - Plugin name: `stdout` -- Flag namespace: [stdout](../2-Deployment/2-Configuration.md#stdout) +- Flag namespace: [stdout](../3-Deployment/2-Configuration.md#stdout) With the stdout plugin, osquery result and/or status logs are written to stdout on the Fleet server. This is typically used for debugging or with a log diff --git a/docs/1-Using-Fleet/6-Security-best-practices.md b/docs/1-Using-Fleet/6-Security-best-practices.md index f3a54d0345..f3ff1acf19 100644 --- a/docs/1-Using-Fleet/6-Security-best-practices.md +++ b/docs/1-Using-Fleet/6-Security-best-practices.md @@ -27,7 +27,7 @@ Fleet supports SAML auth which means that it can be configured such that it neve Passwords are never stored in plaintext in the database. We store a `bcrypt`ed hash of the password along with a randomly generated salt. The `bcrypt` iteration count and salt key size are admin-configurable. ### Authentication tokens -The size and expiration time of session tokens is admin-configurable. See [https://github.com/fleetdm/fleet/blob/master/docs/2-Deployment/2-Configuration.md#session_duration](../2-Deployment/2-Configuration.md#session_duration). +The size and expiration time of session tokens is admin-configurable. See [https://github.com/fleetdm/fleet/blob/master/docs/3-Deployment/2-Configuration.md#session_duration](../3-Deployment/2-Configuration.md#session_duration). It is possible to revoke all session tokens for a user by forcing a password reset. diff --git a/docs/1-Using-Fleet/7-Updating-Fleet.md b/docs/1-Using-Fleet/7-Updating-Fleet.md index 842693b552..1bdee324e5 100644 --- a/docs/1-Using-Fleet/7-Updating-Fleet.md +++ b/docs/1-Using-Fleet/7-Updating-Fleet.md @@ -7,7 +7,7 @@ ## Overview -This guide explains how to update and run new versions of Fleet. For initial installation instructions, see [Installing Fleet](../2-Deployment/1-Installation.md). +This guide explains how to update and run new versions of Fleet. For initial installation instructions, see [Installing Fleet](../3-Deployment/1-Installation.md). There are two steps to perform a typical Fleet update. If any other steps are required, they will be noted in the release notes. diff --git a/docs/1-Using-Fleet/FAQ.md b/docs/1-Using-Fleet/FAQ.md index bfc6db9efb..5b4d09dfd2 100644 --- a/docs/1-Using-Fleet/FAQ.md +++ b/docs/1-Using-Fleet/FAQ.md @@ -17,7 +17,7 @@ It’s standard deployment practice to have multiple Fleet servers behind a load ## How often do labels refresh? Is the refresh frequency configurable? -The update frequency for labels is configurable with the [—osquery_label_update_interval](https://github.com/fleetdm/fleet/blob/master/docs/2-Deployment/2-Configuration.md#osquery_label_update_interval) flag (default 1 hour). +The update frequency for labels is configurable with the [—osquery_label_update_interval](https://github.com/fleetdm/fleet/blob/master/docs/3-Deployment/2-Configuration.md#osquery_label_update_interval) flag (default 1 hour). ## How do I revoke the authorization tokens for a user? @@ -45,7 +45,7 @@ Live query results (executed in the web UI or `fleetctl query`) are pushed direc ### Scheduled Queries -Scheduled query results (queries that are scheduled to run in Packs) are typically sent to the Fleet server, and will be available on the filesystem of the server at the path configurable by [`--osquery_result_log_file`](../2-Deployment/2-Configuration.md#osquery_result_log_file). This defaults to `/tmp/osquery_result`. +Scheduled query results (queries that are scheduled to run in Packs) are typically sent to the Fleet server, and will be available on the filesystem of the server at the path configurable by [`--osquery_result_log_file`](../3-Deployment/2-Configuration.md#osquery_result_log_file). This defaults to `/tmp/osquery_result`. It is possible to configure osqueryd to log query results outside of Fleet. For results to go to Fleet, the `--logger_plugin` flag must be set to `tls`. @@ -53,7 +53,7 @@ It is possible to configure osqueryd to log query results outside of Fleet. For Folks typically use Fleet to ship logs to data aggregation systems like Splunk, the ELK stack, and Graylog. -The [logger configuration options](https://github.com/fleetdm/fleet/blob/master/docs/2-Deployment/2-Configuration.md#osquery_status_log_plugin) allow you to select the log output plugin. Using the log outputs you can route the logs to your chosen aggregation system. +The [logger configuration options](https://github.com/fleetdm/fleet/blob/master/docs/3-Deployment/2-Configuration.md#osquery_status_log_plugin) allow you to select the log output plugin. Using the log outputs you can route the logs to your chosen aggregation system. ### Troubleshooting @@ -63,7 +63,7 @@ Expecting results, but not seeing anything in the logs? - Check whether the query is scheduled in differential mode. If so, new results will only be logged when the result set changes. - Ensure that the query is scheduled to run on the intended platforms, and that the tables queried are supported by those platforms. - Use live query to `SELECT * FROM osquery_schedule` to check whether the query has been scheduled on the host. -- Look at the status logs provided by osquery. In a standard configuration these are available on the filesystem of the Fleet server at the path configurable by [`--filesystem_status_log_file`](../2-Deployment/2-Configuration.md#filesystem_status_log_file). This defaults to `/tmp/osquery_status`. The host will output a status log each time it executes the query. +- Look at the status logs provided by osquery. In a standard configuration these are available on the filesystem of the Fleet server at the path configurable by [`--filesystem_status_log_file`](../3-Deployment/2-Configuration.md#filesystem_status_log_file). This defaults to `/tmp/osquery_status`. The host will output a status log each time it executes the query. ## Why aren’t my live queries being logged? diff --git a/docs/2-Orbit-osquery/README.md b/docs/2-Orbit-osquery/README.md new file mode 100644 index 0000000000..bc094ae52c --- /dev/null +++ b/docs/2-Orbit-osquery/README.md @@ -0,0 +1,152 @@ +# Orbit osquery + +Orbit is an [osquery](https://github.com/osquery/osquery) runtime and autoupdater. With Orbit, it's easy to deploy osquery, manage configurations, and stay up to date. Orbit eases the deployment of osquery connected with a [Fleet server](https://github.com/fleetdm/fleet), and is a (near) drop-in replacement for osquery in a variety of deployment scenarios. + +Orbit is the recommended agent for Fleet. But Orbit can be used with or without Fleet, and Fleet can be used with or without Orbit. + +## Usage + +General information and flag documentation can be accessed by running `orbit --help`. + +### Permissions + +Orbit generally expects root permissions to be able to create and access it's working files. + +To get root level permissions: + +#### macOS/Linux + +Prefix `orbit` commands with `sudo` (`sudo orbit ...`) or run in a root shell. + +#### Windows + +Run Powershell or cmd.exe with "Run as administrator" and start `orbit` commands from that shell. + +### Osquery shell + +Run an `osqueryi` shell with `orbit osqueryi` or `orbit shell`. + +### Connect to a Fleet server + +Use the `--fleet-url` and `--enroll-secret` flags to connect to a Fleet server. + +For example: + +``` sh +orbit --fleet-url=https://localhost:8080 --enroll-secret=the_secret_value +``` + +Use `--fleet_certificate` to provide a path to a certificate bundle when necessary for osquery to verify the authenticity of the Fleet server (typically when using a Windows client or self-signed certificates): + +``` sh +orbit --fleet-url=https://localhost:8080 --enroll-secret=the_secret_value --fleet-certificate=cert.pem +``` + +Add the `--insecure` flag for connections using otherwise invalid certificates: + +``` sh +orbit --fleet-url=https://localhost:8080 --enroll-secret=the_secret_value --insecure +``` + +### Osquery flags + +Orbit can be used as near drop-in replacement for `osqueryd`, enhancing standard osquery with autoupdate capabilities. Orbit passes through any options after `--` directly to the `osqueryd` instance. + +For example, the following would be a typical drop-in usage of Orbit: + +``` sh +orbit -- --flagfile=flags.txt +``` + +## Packaging + +Orbit, like standalone osquery, is typically deployed via OS-specific packages. Tooling is provided with this repository to generate installation packages. + +### Packaging support + +- **macOS** - `.pkg` package generation with (optional) [Notarization](https://developer.apple.com/documentation/xcode/notarizing_macos_software_before_distribution) and codesigning - Persistence via `launchd`. + +- **Linux** - `.deb` (Debian, Ubuntu, etc.) & `.rpm` (RHEL, CentOS, etc.) package generation - Persistence via `systemd`. + +- **Windows** (coming soon) - `.msi` package generation - Persistence via Services. + +### Building packages + +Before building packages, clone or download this repository and [install Go](https://golang.org/doc/install). + +Use `go run ./cmd/package` from this directory to run the packaging tools. + +The only required parameter is `--type`, use one of `deb`, `rpm`, or `pkg` (`msi` coming soon). + +Configure osquery to connect to a Fleet (or other TLS) server with the `--fleet-url` and `--enroll-secret` flags. + +A minimal invocation for communicating with Fleet: + +``` sh +go run ./cmd/package --type deb --fleet-url=fleet.example.com --enroll-secret=notsosecret +``` + +This will build a `.deb` package configured to communicate with a Fleet server at `fleet.example.com` using the enroll secret `notsosecret`. + +When the Fleet server uses a self-signed (or otherwise invalid) TLS certificate, package with the `--insecure` or `--fleet-certificate` options. + +See `go run ./cmd/package` for the full range of packaging options. + +#### Update channels + +Orbit uses the concept of "update channels" to determine the version of Orbit, osquery, and any extensions (extension support coming soon) to run. This concept is modeled from the common versioning convention for Docker containers. + +Configure update channels for Orbit and osqueryd with the `--orbit-channel` and `--osqueryd-channel` flags when packaging. + +| Channel | Versions | +| ------------------------------------ | ------ | +| `4` | 4.x.x | +| `4.6` | 4.6.x | +| `4.6.0` | 4.6.0 | + +Additionally `stable` and `edge` are special channel names. `stable` will always return the version Fleet deems to be stable, while `edge` will provide newer releases for beta testing. + +#### macOS signing & Notarization + +Orbit's packager can automate the codesigning and Notarization steps to allow the resulting package to generate packages that appear "trusted" when install on macOS hosts. Signing & notarization are supported only on macOS hosts. + +For signing, a "Developer ID Installer" certificate must be available on the build machine ([generation instructions](https://help.apple.com/xcode/mac/current/#/dev154b28f09)). Use `security find-identity -v` to verify the existence of this certificate and make note of the identifier provided in the left column. + +For Notarization, valid App Store Connect credentials must be available on the build machine. Set these in the environment variables `AC_USERNAME` and `AC_PASSWORD`. It is common to configure this via [app-specific passwords](https://support.apple.com/en-ca/HT204397). + +Build a signed and notarized macOS package with an invocation like the following: + +``` sh +AC_USERNAME=zach@fleetdm.com AC_PASSWORD=llpk-sije-kjlz-jdzw go run ./cmd/package --type=pkg --fleet-url=fleet.example.com --enroll-secret=63SBzTT+2UyW --sign-identity 3D7260BF99539C6E80A94835A8921A988F4E6498 --notarize +``` + +This process may take several minutes to complete as the Notarization process completes on Apple's servers. + +After successful notarization, the generated "ticket" is automatically stapled to the package. + +## FAQs + +### How does Orbit compare with Kolide Launcher? + +Orbit is inspired by the success of [Kolide Launcher](https://github.com/kolide/launcher), and approaches a similar problem domain with new strategies informed by the challenges encountered in real world deployments. Orbit does not share any code with Launcher. + +- Both Orbit and Launcher use [The Update Framework](https://theupdateframework.com/) specification for managing updates. Orbit utilizes the official [go-tuf](https://github.com/theupdateframework/go-tuf) library, while Launcher has it's own implementation of the specification. +- Orbit can be deployed as a (near) drop-in replacement for osquery, supporting full customization of the osquery flags. Launcher heavily manages the osquery flags making deployment outside of Fleet or Kolide's SaaS difficult. +- Orbit prefers the battle-tested plugins of osquery. Orbit uses the built-in logging, configuration, and live query plugins, while Launcher uses custom implementations. +- Orbit prefers the built-in osquery remote APIs. Launcher utilizes a custom gRPC API that has led to issues with character encoding, load balancers/proxies, and request size limits. +- Orbit encourages use of the osquery performance Watchdog, while Launcher disables the Watchdog. + +Additionally, Orbit aims to tackle problems out of scope for Launcher: + +- Configure updates via release channels, providing more granular control over agent versioning. +- Support for deploying and updating osquery extensions (🔜). +- Manage osquery versions and startup flags from a remote (Fleet) server (🔜). +- Further control of osquery performance via cgroups (🔜). + +### Is Orbit Free? + +Yes! Orbit is licensed under an MIT license and all uses are encouraged. + +### How does orbit update osquery? And how do the stable and edge channels get triggered to update osquery on a self hosted Fleet instance? + +Orbit uses a configurable update server. We expect that many folks will just use the update server we manage (similar to what Kolide does with Launcher's update server). We are also offering [tooling for self-managing an update server](https://github.com/fleetdm/fleet/blob/master/docs/3-Deployment/4-fleetctl-agent-updates.md) as part of Fleet Basic (the subscription offering). diff --git a/docs/2-Deployment/1-Installation.md b/docs/3-Deployment/1-Installation.md similarity index 100% rename from docs/2-Deployment/1-Installation.md rename to docs/3-Deployment/1-Installation.md diff --git a/docs/2-Deployment/2-Configuration.md b/docs/3-Deployment/2-Configuration.md similarity index 100% rename from docs/2-Deployment/2-Configuration.md rename to docs/3-Deployment/2-Configuration.md diff --git a/docs/2-Deployment/3-Adding-hosts.md b/docs/3-Deployment/3-Adding-hosts.md similarity index 100% rename from docs/2-Deployment/3-Adding-hosts.md rename to docs/3-Deployment/3-Adding-hosts.md diff --git a/docs/2-Deployment/4-Example-deployment-scenarios.md b/docs/3-Deployment/4-Example-deployment-scenarios.md similarity index 100% rename from docs/2-Deployment/4-Example-deployment-scenarios.md rename to docs/3-Deployment/4-Example-deployment-scenarios.md diff --git a/docs/2-Deployment/4-fleetctl-agent-updates.md b/docs/3-Deployment/4-fleetctl-agent-updates.md similarity index 100% rename from docs/2-Deployment/4-fleetctl-agent-updates.md rename to docs/3-Deployment/4-fleetctl-agent-updates.md diff --git a/docs/2-Deployment/FAQ.md b/docs/3-Deployment/FAQ.md similarity index 100% rename from docs/2-Deployment/FAQ.md rename to docs/3-Deployment/FAQ.md diff --git a/docs/2-Deployment/README.md b/docs/3-Deployment/README.md similarity index 100% rename from docs/2-Deployment/README.md rename to docs/3-Deployment/README.md diff --git a/docs/3-Contribution/1-Building-Fleet.md b/docs/4-Contribution/1-Building-Fleet.md similarity index 100% rename from docs/3-Contribution/1-Building-Fleet.md rename to docs/4-Contribution/1-Building-Fleet.md diff --git a/docs/3-Contribution/2-Testing.md b/docs/4-Contribution/2-Testing.md similarity index 100% rename from docs/3-Contribution/2-Testing.md rename to docs/4-Contribution/2-Testing.md diff --git a/docs/3-Contribution/3-Migrations.md b/docs/4-Contribution/3-Migrations.md similarity index 100% rename from docs/3-Contribution/3-Migrations.md rename to docs/4-Contribution/3-Migrations.md diff --git a/docs/3-Contribution/4-Committing-Changes.md b/docs/4-Contribution/4-Committing-Changes.md similarity index 100% rename from docs/3-Contribution/4-Committing-Changes.md rename to docs/4-Contribution/4-Committing-Changes.md diff --git a/docs/3-Contribution/5-Releasing-Fleet.md b/docs/4-Contribution/5-Releasing-Fleet.md similarity index 100% rename from docs/3-Contribution/5-Releasing-Fleet.md rename to docs/4-Contribution/5-Releasing-Fleet.md diff --git a/docs/3-Contribution/FAQ.md b/docs/4-Contribution/FAQ.md similarity index 100% rename from docs/3-Contribution/FAQ.md rename to docs/4-Contribution/FAQ.md diff --git a/docs/3-Contribution/README.md b/docs/4-Contribution/README.md similarity index 100% rename from docs/3-Contribution/README.md rename to docs/4-Contribution/README.md diff --git a/docs/README.md b/docs/README.md index 68b906da59..6e152ad0d5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -3,12 +3,15 @@ Welcome to the documentation for the Fleet osquery fleet manager. ### [Using Fleet](./1-Using-Fleet/README.md) -Resources for using the Fleet UI, fleetctl CLI, and Fleet REST API +Resources for using the Fleet UI, fleetctl CLI, and Fleet REST API. -### [Deployment](./2-Deployment/README.md) -Resources for installing Fleet's infrastructure dependencies, configuring Fleet, deploying osquery to hosts, and viewing example deployment scenarios +### [Orbit osquery](./2-Orbit-osquery/README.md) +Resources for using Orbit for osquery: the recommended agent for Fleet. -### [Contribution](./3-Contribution/README.md) +### [Deployment](./3-Deployment/README.md) +Resources for installing Fleet's infrastructure dependencies, configuring Fleet, deploying osquery to hosts, and viewing example deployment scenarios. + +### [Contribution](./4-Contribution/README.md) If you're interested in interacting with the Fleet source code, you'll find information on modifying and building the code here. --- diff --git a/frontend/README.md b/frontend/README.md index 819ba2439e..44fee0a772 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -5,7 +5,7 @@ The Fleet front-end is a Single Page Application using React and Redux. ## Running the Fleet web app For details instruction on building and serving the Fleet web application -consult the [Development Documentation](https://github.com/fleetdm/fleet/tree/master/docs/2-Deployment) +consult the [Development Documentation](https://github.com/fleetdm/fleet/tree/master/docs/3-Deployment) ## Directory Structure diff --git a/frontend/components/hosts/AddHostModal/AddHostModal.jsx b/frontend/components/hosts/AddHostModal/AddHostModal.jsx index de315aec4e..62998673a7 100644 --- a/frontend/components/hosts/AddHostModal/AddHostModal.jsx +++ b/frontend/components/hosts/AddHostModal/AddHostModal.jsx @@ -121,7 +121,7 @@ class AddHostModal extends Component {

diff --git a/tools/backup_db/README.md b/tools/backup_db/README.md index 928c706e72..3a6dc69d93 100644 --- a/tools/backup_db/README.md +++ b/tools/backup_db/README.md @@ -1,3 +1,3 @@ These scripts are for backing up and restore the Docker development MySQL database. -Usage is documented [here](../../docs/3-Contribution/2-Testing.md#database-backuprestore). +Usage is documented [here](../../docs/4-Contribution/2-Testing.md#database-backuprestore). diff --git a/website/config/routes.js b/website/config/routes.js index be89883e3d..3da29d5e5e 100644 --- a/website/config/routes.js +++ b/website/config/routes.js @@ -36,7 +36,7 @@ module.exports.routes = { 'GET /pricing': { action: 'view-pricing' }, '/try-fleet': '/get-started', '/documentation': 'https://github.com/fleetdm/fleet/tree/master/docs', - '/contribute': 'https://github.com/fleetdm/fleet/tree/master/docs/3-Contribution', + '/contribute': 'https://github.com/fleetdm/fleet/tree/master/docs/4-Contribution', '/hall-of-fame': 'https://github.com/fleetdm/fleet/pulse', From 7c88de8f499afbe683ae7e17ea3588dcdba7ef5e Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Wed, 5 May 2021 10:39:11 -0500 Subject: [PATCH 06/51] better experience for infrequent npm users --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ced3e6e7be..b95e51a7af 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Fleet is the most widely used open source osquery manager. Deploying osquery wi ```bash # Install the Fleet command-line tool -npm install -g fleetctl +sudo npm install -g fleetctl # Run a local demo of the Fleet server sudo fleetctl preview ``` From ea051952009519ddb8ba1538de50a90958b14566 Mon Sep 17 00:00:00 2001 From: noahtalerman <47070608+noahtalerman@users.noreply.github.com> Date: Wed, 5 May 2021 18:00:05 -0400 Subject: [PATCH 07/51] Add questions to FAQ and add documentation section to CONTRIBUTING.md (#726) - Add section on contributing to Fleet documentation. This section covers the use of relative links and best practices for anchor links - Add 3 questions and their respective answers to the FAQ section in the `1-Using-Fleet` docs --- CONTRIBUTING.md | 20 ++++++++++++++++++++ docs/1-Using-Fleet/FAQ.md | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fe194fc8d6..64b8de7bde 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,3 +33,23 @@ To open your browser's **network requests**, press Control Shift J (Windows, Lin Sensitive security-related issues should be reported to [security@fleetdm.com](mailto:security@fleetdm.com) before a public issue is made. + +## Contributing to documentation + +Fleet currently uses GitHub as the central location for documentation. + +### Markdown links + +Due to the structure of the Fleet documentation and GitHub's unique markdown files, there are several practices we'd like to call out if you're documentation changes include links to other locations within the Fleet docs. + +#### Relative links + +When including a link to a different file in the Fleet documentation please use relative links when possible. + +For example, let's say you're working on changes in the Contribution docs and you'd like to add a link to the REST API docs. The relative link would look something like `../1-Using-Fleet/3-REST-API.md`. + +#### Special characters in anchor links + +There are certain characters GitHub doesn't support in the use of anchor links in markdown files. The general rule we've found is to only use a-z or A-Z characters in anchor links. All other characters should be removed. + +For example, consider the section title *How do I connect to the Mailhog simulated server?*. The valid GitHub anchor link for this section is #how-do-i-connect-to-the-mailhog-simulated-server. Notice the *?* character is removed. diff --git a/docs/1-Using-Fleet/FAQ.md b/docs/1-Using-Fleet/FAQ.md index 5b4d09dfd2..125c1228bf 100644 --- a/docs/1-Using-Fleet/FAQ.md +++ b/docs/1-Using-Fleet/FAQ.md @@ -1,13 +1,16 @@ # Using Fleet FAQ - [Has anyone stress tested Fleet? How many clients can the Fleet server handle?](#has-anyone-stress-tested-fleet-how-many-clients-can-the-fleet-server-handle) +- [Can I target my hosts using their enroll secrets?](#can-I-target-my-hosts-using-their-enroll-secrets) - [How often do labels refresh? Is the refresh frequency configurable?](#how-often-do-labels-refresh-is-the-refresh-frequency-configurable) - [How do I revoke the authorization tokens for a user?](#how-do-i-revoke-the-authorization-tokens-for-a-user) - [How do I monitor the performance of my queries?](#how-do-i-monitor-the-performance-of-my-queries) - [How do I monitor a Fleet server?](#how-do-i-monitor-a-fleet-server) - [Why is the “Add User” button disabled?](#why-is-the-add-user-button-disabled) +- [Can I disable password-based authentication in the Fleet UI?](#can-i-disable-password-based-authentication-in-the-fleet-ui) - [Where are my query results?](#where-are-my-query-results) - [Why aren’t my live queries being logged?](#why-arent-my-live-queries-being-logged) +- [Can I use the Fleet API to fetch results from a scheduled query pack?](#can-i-use-the-fleet-api-to-fetch-results-from-a-scheduled-query-pack) ## Has anyone stress tested Fleet? How many clients can the Fleet server handle? @@ -15,6 +18,14 @@ Fleet has been stress tested to 150,000 online hosts and 400,000 total enrolled It’s standard deployment practice to have multiple Fleet servers behind a load balancer. However, typically the MySQL database is the performance bottleneck and a single Fleet server can handle tens of thousands of hosts. +## Can I target my hosts using their enroll secrets? + +No, currently, there’s no way to retrieve the name of the enroll secret with a query. This means that there's no way to create a label using your hosts' enroll secrets and then use this label as a target for queries or query packs. + +Typically folks will use some other unique identifier to create labels that distinguish each type of device. As a workaround, [Fleet's manual labels](./2-fleetctl-CLI.md#host-labels) provide a way to create groups of hosts without a query. These manual labels can then be used as targets for queries or query packs. + +In the coming months, Fleet will introduce the [Teams feature](https://github.com/fleetdm/fleet/issues/288) to Fleet Basic customers. Teams will allow users to group hosts into teams using their enroll secret. These teams can then be targeted with specific queries, query packs, and configuration. + ## How often do labels refresh? Is the refresh frequency configurable? The update frequency for labels is configurable with the [—osquery_label_update_interval](https://github.com/fleetdm/fleet/blob/master/docs/3-Deployment/2-Configuration.md#osquery_label_update_interval) flag (default 1 hour). @@ -37,6 +48,15 @@ The “Add User” button is disabled if SMTP (email) has not been configured fo One way to hack around this is to use a simulated mailserver like [Mailhog](https://github.com/mailhog/MailHog). You can retrieve the email that was “sent” in the Mailhog UI, and provide users with the invite URL manually. +## Can I disable password-based authentication in the Fleet UI? + +Some folks like to enforce users with SAML SSO enabled to login only via the SSO and not via password. + +There is no option in the Fleet UI for disabling password-based authentication. +However, users that have SSO enabled in Fleet will not be able to log in via password-based authentication. + +If a user has SSO enabled, the Login page in the Fleet UI does displays the “Username” and “Password” fields but on attempted password-based login, this user will receive an “Authentication failed” message. + ## Where are my query results? ### Live Queries @@ -68,3 +88,19 @@ Expecting results, but not seeing anything in the logs? ## Why aren’t my live queries being logged? Live query results are never logged to the filesystem of the Fleet server. See [Where are my query results?](#where-are-my-query-results). + +## Can I use the Fleet API to fetch results from a scheduled query pack? + +You cannot. Scheduled query results are logged to whatever logging plugin you have configured and are not stored in the Fleet DB. + +However, the Fleet API exposes a significant amount of host information via the [`api/v1/fleet/hosts`](./3-REST-API.md#list-hosts) and the [`api/v1/fleet/hosts/{id}`](./3-REST-API.md#get-host) API endpoints. The `api/v1/fleet/hosts` [can even be configured to return additional host information](https://github.com/fleetdm/fleet/blob/9fb9da31f5462fa7dda4819a114bbdbc0252c347/docs/1-Using-Fleet/2-fleetctl-CLI.md#fleet-configuration-options). + +As an example, let's say you want to retrieve a host's OS version, installed software, and kernel version: + +Each host’s OS version is available using the `api/v1/fleet/hosts` API endpoint. [Check out the API documentation for this endpoint](./3-REST-API.md#list-hosts). + +The ability to view each host’s installed software was released behind a feature flag in Fleet 3.11.0 and called Software inventory. [Check out the feature flag documentation for instructions on turning on Software inventory in Fleet](../3-Deployment/2-Configuration.md#feature-flags). + +Once the Software inventory feature is turned on, a list of a specific host’s installed software is available using the `api/v1/fleet/hosts/{id}` endpoint. [Check out the documentation for this endpoint](./3-REST-API.md#get-host). + +It’s possible in Fleet to retrieve each host’s kernel version, using the Fleet API, through `additional_queries`. The Fleet configuration options yaml file includes an `additional_queries` property that allows you to append custom query results to the host details returned by the `api/v1/fleet/hosts` endpoint. [Check out an example configuration file with the additional_queries field](./2-fleetctl-CLI.md#fleet-configuration-options). \ No newline at end of file From 26c841bcd5e8650ebf63a278b0a7a9dc8f72bbce Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Wed, 5 May 2021 21:27:58 -0500 Subject: [PATCH 08/51] update calendly links & dedupe quickstart for now (#728) * update calendly link * update other calendly link * consolidate /get-started * Update routes.js * demux to /get-started * lint fix * lint fix --- website/config/routes.js | 19 +++++++------------ website/views/layouts/layout.ejs | 4 ++-- website/views/pages/get-started.ejs | 2 +- website/views/pages/homepage.ejs | 2 +- website/views/pages/pricing.ejs | 2 +- 5 files changed, 12 insertions(+), 17 deletions(-) diff --git a/website/config/routes.js b/website/config/routes.js index 3da29d5e5e..a0c5b6e87e 100644 --- a/website/config/routes.js +++ b/website/config/routes.js @@ -17,24 +17,15 @@ module.exports.routes = { '/blog': 'https://medium.com/fleetdm', - '/pricing': (req, res)=>{ - // FUTURE: pricing page explaining commercial support and EE, w/ our subscription terms - res.redirect('mailto:todo@example.com?subject=Pricing question&body=Please do not send this email!\n\nWe are a very young company and still working on our processes. For now, if you have a pricing question or would like to know Fleet\'s latest pricing and support tiers, please create an issue at https://github.com/fleetdm/fleet/issues. Thank you!'); - }, - '/legal/terms': 'https://docs.google.com/document/d/1OM6YDVIs7bP8wg6iA3VG13X086r64tWDqBSRudG4a0Y/edit', - '/security': (req, res)=>{ - // FUTURE: make a page- check out how Sails does it, and also https://about.gitlab.com/security/ - res.redirect('mailto:todo@example.com?subject=Security vulnerability&body=Please do not send this email!\n\nWe are a very young company and still working on our processes. For now, if you have a security vulnerability to report, please send a DM to mikermcneil or Zach Wasserman in the "osquery" Slack workspace. Thank you for letting us know!'); - }, + '/security': 'https://github.com/fleetdm/fleet/security/policy', '/company/about': '/blog', // FUTURE: brief "about" page explaining the origins of the company '/company/stewardship': 'https://github.com/fleetdm/fleet', // FUTURE: page about how we approach open source and our commitments to the community 'GET /company/contact': { action: 'view-contact' }, - 'GET /get-started': { action: 'view-get-started' }, + 'GET /get-started': 'https://github.com/fleetdm/fleet/blob/master/README.md', // « FUTURE: When ready, bring back { action: 'view-get-started' } 'GET /pricing': { action: 'view-pricing' }, - '/try-fleet': '/get-started', '/documentation': 'https://github.com/fleetdm/fleet/tree/master/docs', '/contribute': 'https://github.com/fleetdm/fleet/tree/master/docs/4-Contribution', '/hall-of-fame': 'https://github.com/fleetdm/fleet/pulse', @@ -62,13 +53,17 @@ module.exports.routes = { // ╔╦╗╦╔═╗╔═╗ ╦═╗╔═╗╔╦╗╦╦═╗╔═╗╔═╗╔╦╗╔═╗ ┬ ╔╦╗╔═╗╦ ╦╔╗╔╦ ╔═╗╔═╗╔╦╗╔═╗ // ║║║║╚═╗║ ╠╦╝║╣ ║║║╠╦╝║╣ ║ ║ ╚═╗ ┌┼─ ║║║ ║║║║║║║║ ║ ║╠═╣ ║║╚═╗ // ╩ ╩╩╚═╝╚═╝ ╩╚═╚═╝═╩╝╩╩╚═╚═╝╚═╝ ╩ ╚═╝ └┘ ═╩╝╚═╝╚╩╝╝╚╝╩═╝╚═╝╩ ╩═╩╝╚═╝ - // '/logout': '/api/v1/account/logout', + + // Convenience '/company': '/company/about', '/support': '/company/contact', '/contact': '/company/contact', '/legal': '/legal/terms', '/terms': '/legal/terms', + // '/logout': '/api/v1/account/logout', + // Legacy (to avoid breaking links) + '/try-fleet': '/get-started', // ╦ ╦╔═╗╔╗ ╦ ╦╔═╗╔═╗╦╔═╔═╗ // ║║║║╣ ╠╩╗╠═╣║ ║║ ║╠╩╗╚═╗ diff --git a/website/views/layouts/layout.ejs b/website/views/layouts/layout.ejs index dbe7c70c92..a6e9da0944 100644 --- a/website/views/layouts/layout.ejs +++ b/website/views/layouts/layout.ejs @@ -81,7 +81,7 @@
Get started - Try Fleet + Try Fleet Documentation Contribute Hall of fame @@ -102,7 +102,7 @@ Get started
- Get started with Fleet + Get started with Fleet

diff --git a/website/views/pages/pricing.ejs b/website/views/pages/pricing.ejs index a521575268..41099c9064 100644 --- a/website/views/pages/pricing.ejs +++ b/website/views/pages/pricing.ejs @@ -1073,7 +1073,7 @@ Date: Wed, 5 May 2021 22:07:15 -0500 Subject: [PATCH 09/51] Set up fleetdm.com/apply to list of openings (#729) --- website/config/routes.js | 1 + 1 file changed, 1 insertion(+) diff --git a/website/config/routes.js b/website/config/routes.js index a0c5b6e87e..98907fa605 100644 --- a/website/config/routes.js +++ b/website/config/routes.js @@ -24,6 +24,7 @@ module.exports.routes = { '/company/about': '/blog', // FUTURE: brief "about" page explaining the origins of the company '/company/stewardship': 'https://github.com/fleetdm/fleet', // FUTURE: page about how we approach open source and our commitments to the community 'GET /company/contact': { action: 'view-contact' }, + 'GET /apply': 'https://fleet-device-management.breezy.hr', 'GET /get-started': 'https://github.com/fleetdm/fleet/blob/master/README.md', // « FUTURE: When ready, bring back { action: 'view-get-started' } 'GET /pricing': { action: 'view-pricing' }, '/documentation': 'https://github.com/fleetdm/fleet/tree/master/docs', From c67d9bc8ebccd11025a75797e62b98f6c0299387 Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Thu, 6 May 2021 13:46:52 -0500 Subject: [PATCH 10/51] Send contact form messages to Slack (#733) * bring back action to handle contact form * deliver contact form messages to Slack. --- .../deliver-contact-form-message.js | 24 +++---------------- website/config/custom.js | 3 ++- website/config/routes.js | 2 +- 3 files changed, 6 insertions(+), 23 deletions(-) diff --git a/website/api/controllers/deliver-contact-form-message.js b/website/api/controllers/deliver-contact-form-message.js index 7668439757..4e04aed81c 100644 --- a/website/api/controllers/deliver-contact-form-message.js +++ b/website/api/controllers/deliver-contact-form-message.js @@ -50,27 +50,9 @@ module.exports = { fn: async function({emailAddress, topic, fullName, message}) { - if (!sails.config.custom.internalEmailAddress) { - throw new Error( -`Cannot deliver incoming message from contact form because there is no internal -email address (\`sails.config.custom.internalEmailAddress\`) configured for this -app. To enable contact form emails, you'll need to add this missing setting to -your custom config -- usually in \`config/custom.js\`, \`config/staging.js\`, -\`config/production.js\`, or via system environment variables.` - ); - } - - await sails.helpers.sendTemplateEmail.with({ - to: sails.config.custom.internalEmailAddress, - subject: 'New contact form message', - template: 'internal/email-contact-form', - layout: false, - templateData: { - contactName: fullName, - contactEmail: emailAddress, - topic, - message, - } + await sails.helpers.http.post(sails.config.custom.slackWebhookUrlForContactForm, { + text: `New contact form message: (Remember: we have to email back; can't just reply to this thread.) cc @sales `+ + `Name: ${fullName}, Email: ${emailAddress}, Topic: ${topic}, Message: ${message}` }); } diff --git a/website/config/custom.js b/website/config/custom.js index 44de0294c5..fa8de14f08 100644 --- a/website/config/custom.js +++ b/website/config/custom.js @@ -98,8 +98,9 @@ module.exports.custom = { /*************************************************************************** * * * Any other custom config this Sails app should use during development. * + * (and possibly in ALL environments, if not overridden in config/env/) * * * ***************************************************************************/ - // … + //… }; diff --git a/website/config/routes.js b/website/config/routes.js index 98907fa605..8173d80c06 100644 --- a/website/config/routes.js +++ b/website/config/routes.js @@ -85,6 +85,6 @@ module.exports.routes = { // 'POST /api/v1/entrance/signup': { action: 'entrance/signup' }, // 'POST /api/v1/entrance/send-password-recovery-email': { action: 'entrance/send-password-recovery-email' }, // 'POST /api/v1/entrance/update-password-and-login': { action: 'entrance/update-password-and-login' }, - // 'POST /api/v1/deliver-contact-form-message': { action: 'deliver-contact-form-message' }, + 'POST /api/v1/deliver-contact-form-message': { action: 'deliver-contact-form-message' }, }; From b0af4287942557d27499392f86c916f395ca032a Mon Sep 17 00:00:00 2001 From: Mike Thomas <78363703+mike-j-thomas@users.noreply.github.com> Date: Fri, 7 May 2021 06:48:00 +0900 Subject: [PATCH 11/51] Added osquery link to testimonial (#730) * Added osquery link to testimonial * Update homepage.ejs Changed inline-block style to d-inline-block class. --- website/views/pages/homepage.ejs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/views/pages/homepage.ejs b/website/views/pages/homepage.ejs index 80423fbeb3..11dfd62d8a 100644 --- a/website/views/pages/homepage.ejs +++ b/website/views/pages/homepage.ejs @@ -51,8 +51,8 @@
From f788254e61c4e3a9539e9693f573b0669ec5f7f8 Mon Sep 17 00:00:00 2001 From: Zach Wasserman Date: Thu, 6 May 2021 21:05:09 -0700 Subject: [PATCH 12/51] Implement storage of scheduled query statistics (#735) Track all data from the osquery_schedule table on a per-host basis. This data is now returned when retrieving host details in the API. --- server/datastore/datastore.go | 1 + server/datastore/datastore_hosts.go | 119 ++++++++++++ .../datastore/datastore_scheduled_queries.go | 8 +- server/datastore/mysql/hosts.go | 128 ++++++++++++- .../20210506095025_AddScheduledQueryStats.go | 40 ++++ server/datastore/mysql/scheduled_queries.go | 2 +- server/kolide/hosts.go | 3 + server/kolide/packs.go | 6 + server/kolide/scheduled_queries.go | 23 +++ server/service/service_osquery.go | 74 ++++++++ server/service/service_osquery_test.go | 172 ++++++++++++++++++ server/test/new_objects.go | 9 +- 12 files changed, 578 insertions(+), 7 deletions(-) create mode 100644 server/datastore/mysql/migrations/tables/20210506095025_AddScheduledQueryStats.go diff --git a/server/datastore/datastore.go b/server/datastore/datastore.go index af0bd4beca..b7470e7a79 100644 --- a/server/datastore/datastore.go +++ b/server/datastore/datastore.go @@ -44,6 +44,7 @@ var TestFunctions = [...]func(*testing.T, kolide.Datastore){ testListHostsInLabel, testListUniqueHostsInLabels, testSaveHosts, + testSaveHostPackStats, testDeleteHost, testListHosts, testListHostsFilterAdditional, diff --git a/server/datastore/datastore_hosts.go b/server/datastore/datastore_hosts.go index 7d3c8ff5b8..a00579ec98 100644 --- a/server/datastore/datastore_hosts.go +++ b/server/datastore/datastore_hosts.go @@ -92,6 +92,125 @@ func testSaveHosts(t *testing.T, ds kolide.Datastore) { assert.Nil(t, host) } +func testSaveHostPackStats(t *testing.T, ds kolide.Datastore) { + host, err := ds.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + LabelUpdateTime: time.Now(), + SeenTime: time.Now(), + NodeKey: "1", + UUID: "1", + HostName: "foo.local", + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + }) + require.NoError(t, err) + require.NotNil(t, host) + + // Pack and query must exist for stats to save successfully + pack1 := test.NewPack(t, ds, "test1") + query1 := test.NewQuery(t, ds, "time", "select * from time", 0, true) + squery1 := test.NewScheduledQuery(t, ds, pack1.ID, query1.ID, 30, true, true, "time-scheduled") + stats1 := []kolide.ScheduledQueryStats{ + { + ScheduledQueryName: squery1.Name, + ScheduledQueryID: squery1.ID, + QueryName: query1.Name, + PackName: pack1.Name, + PackID: pack1.ID, + AverageMemory: 8000, + Denylisted: false, + Executions: 164, + Interval: 30, + LastExecuted: time.Unix(1620325191, 0).UTC(), + OutputSize: 1337, + SystemTime: 150, + UserTime: 180, + WallTime: 0, + }, + } + + pack2 := test.NewPack(t, ds, "test2") + squery2 := test.NewScheduledQuery(t, ds, pack2.ID, query1.ID, 30, true, true, "time-scheduled") + query2 := test.NewQuery(t, ds, "processes", "select * from processes", 0, true) + squery3 := test.NewScheduledQuery(t, ds, pack2.ID, query2.ID, 30, true, true, "processes") + stats2 := []kolide.ScheduledQueryStats{ + { + ScheduledQueryName: squery2.Name, + ScheduledQueryID: squery2.ID, + QueryName: query1.Name, + PackName: pack2.Name, + PackID: pack2.ID, + AverageMemory: 431, + Denylisted: true, + Executions: 1, + Interval: 30, + LastExecuted: time.Unix(980943843, 0).UTC(), + OutputSize: 134, + SystemTime: 1656, + UserTime: 18453, + WallTime: 10, + }, + { + ScheduledQueryName: squery3.Name, + ScheduledQueryID: squery3.ID, + QueryName: query2.Name, + PackName: pack2.Name, + PackID: pack2.ID, + AverageMemory: 8000, + Denylisted: false, + Executions: 164, + Interval: 30, + LastExecuted: time.Unix(1620325191, 0).UTC(), + OutputSize: 1337, + SystemTime: 150, + UserTime: 180, + WallTime: 0, + }, + } + + host.PackStats = []kolide.PackStats{ + { + PackName: "test1", + // Append an additional entry to be sure that receiving stats for a + // now-deleted query doesn't break saving. This extra entry should + // not be returned on loading the host. + QueryStats: append(stats1, kolide.ScheduledQueryStats{PackName: "foo", ScheduledQueryName: "bar"}), + }, + { + PackName: "test2", + QueryStats: stats2, + }, + } + + require.NoError(t, ds.SaveHost(host)) + + host, err = ds.Host(host.ID) + require.NoError(t, err) + + require.Len(t, host.PackStats, 2) + sort.Slice(host.PackStats, func(i, j int) bool { + return host.PackStats[i].PackName < host.PackStats[j].PackName + }) + assert.Equal(t, host.PackStats[0].PackName, "test1") + assert.ElementsMatch(t, host.PackStats[0].QueryStats, stats1) + assert.Equal(t, host.PackStats[1].PackName, "test2") + assert.ElementsMatch(t, host.PackStats[1].QueryStats, stats2) + + // Set to nil should not overwrite + host.PackStats = nil + require.NoError(t, ds.SaveHost(host)) + host, err = ds.Host(host.ID) + require.NoError(t, err) + require.Len(t, host.PackStats, 2) + + // Set to empty should make it empty + host.PackStats = []kolide.PackStats{} + require.NoError(t, ds.SaveHost(host)) + host, err = ds.Host(host.ID) + require.NoError(t, err) + require.Len(t, host.PackStats, 0) +} + func testDeleteHost(t *testing.T, ds kolide.Datastore) { host, err := ds.NewHost(&kolide.Host{ DetailUpdateTime: time.Now(), diff --git a/server/datastore/datastore_scheduled_queries.go b/server/datastore/datastore_scheduled_queries.go index caf586c2e0..3525930f4c 100644 --- a/server/datastore/datastore_scheduled_queries.go +++ b/server/datastore/datastore_scheduled_queries.go @@ -84,9 +84,11 @@ func testNewScheduledQuery(t *testing.T, ds kolide.Datastore) { query, err := ds.NewScheduledQuery(&kolide.ScheduledQuery{ PackID: p1.ID, QueryID: q1.ID, + Name: "foo-scheduled", }) require.Nil(t, err) - assert.Equal(t, "foo", query.Name) + assert.Equal(t, "foo", query.QueryName) + assert.Equal(t, "foo-scheduled", query.Name) assert.Equal(t, "select * from time;", query.Query) } @@ -94,7 +96,7 @@ func testScheduledQuery(t *testing.T, ds kolide.Datastore) { u1 := test.NewUser(t, ds, "Admin", "admin", "admin@kolide.co", true) q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true) p1 := test.NewPack(t, ds, "baz") - sq1 := test.NewScheduledQuery(t, ds, p1.ID, q1.ID, 60, false, false) + sq1 := test.NewScheduledQuery(t, ds, p1.ID, q1.ID, 60, false, false, "") query, err := ds.ScheduledQuery(sq1.ID) require.Nil(t, err) @@ -118,7 +120,7 @@ func testDeleteScheduledQuery(t *testing.T, ds kolide.Datastore) { u1 := test.NewUser(t, ds, "Admin", "admin", "admin@kolide.co", true) q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true) p1 := test.NewPack(t, ds, "baz") - sq1 := test.NewScheduledQuery(t, ds, p1.ID, q1.ID, 60, false, false) + sq1 := test.NewScheduledQuery(t, ds, p1.ID, q1.ID, 60, false, false, "") query, err := ds.ScheduledQuery(sq1.ID) require.Nil(t, err) diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index e23ee4cb79..0392761dfa 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -3,6 +3,7 @@ package mysql import ( "database/sql" "fmt" + "strings" "time" "github.com/cenkalti/backoff/v4" @@ -129,6 +130,124 @@ func (d *Datastore) SaveHost(host *kolide.Host) error { return errors.Wrapf(err, "save host with id %d", host.ID) } + // Save host pack stats only if it is non-nil. Empty stats should be + // represented by an empty slice. + if host.PackStats != nil { + if err := d.saveHostPackStats(host); err != nil { + return err + } + } + + return nil +} + +func (d *Datastore) saveHostPackStats(host *kolide.Host) error { + if err := d.withRetryTxx(func(tx *sqlx.Tx) error { + sql := ` + DELETE FROM scheduled_query_stats + WHERE host_id = ? + ` + if _, err := tx.Exec(sql, host.ID); err != nil { + return errors.Wrap(err, "delete old stats") + } + + // Bulk insert software entries + var args []interface{} + queryCount := 0 + for _, pack := range host.PackStats { + for _, query := range pack.QueryStats { + queryCount++ + + args = append(args, + query.PackName, + query.ScheduledQueryName, + host.ID, + query.AverageMemory, + query.Denylisted, + query.Executions, + query.Interval, + query.LastExecuted, + query.OutputSize, + query.SystemTime, + query.UserTime, + query.WallTime, + ) + } + } + + if queryCount == 0 { + return nil + } + + values := strings.TrimSuffix(strings.Repeat("((SELECT sq.id FROM scheduled_queries sq JOIN packs p ON (sq.pack_id = p.id) WHERE p.name = ? AND sq.name = ?),?,?,?,?,?,?,?,?,?,?),", queryCount), ",") + sql = fmt.Sprintf(` + INSERT IGNORE INTO scheduled_query_stats ( + scheduled_query_id, + host_id, + average_memory, + denylisted, + executions, + schedule_interval, + last_executed, + output_size, + system_time, + user_time, + wall_time + ) + VALUES %s + `, values) + if _, err := tx.Exec(sql, args...); err != nil { + return errors.Wrap(err, "insert pack stats") + } + + return nil + }); err != nil { + return errors.Wrap(err, "save pack stats") + } + return nil +} + +func (d *Datastore) loadHostPackStats(host *kolide.Host) error { + sql := ` +SELECT + sqs.scheduled_query_id, + sqs.average_memory, + sqs.denylisted, + sqs.executions, + sqs.schedule_interval, + sqs.last_executed, + sqs.output_size, + sqs.system_time, + sqs.user_time, + sqs.wall_time, + sq.name AS scheduled_query_name, + sq.id AS scheduled_query_id, + sq.query_name AS query_name, + p.name AS pack_name, + p.id as pack_id +FROM scheduled_query_stats sqs + JOIN scheduled_queries sq ON (sqs.scheduled_query_id = sq.id) + JOIN packs p ON (sq.pack_id = p.id) +WHERE host_id = ? +` + var stats []kolide.ScheduledQueryStats + if err := d.db.Select(&stats, sql, host.ID); err != nil { + return errors.Wrap(err, "load pack stats") + } + + packs := map[uint]kolide.PackStats{} + for _, query := range stats { + pack := packs[query.PackID] + pack.PackName = query.PackName + pack.PackID = query.PackID + pack.QueryStats = append(pack.QueryStats, query) + packs[pack.PackID] = pack + } + + for _, pack := range packs { + host.PackStats = append(host.PackStats, pack) + } + return nil } @@ -148,7 +267,10 @@ func (d *Datastore) Host(id uint) (*kolide.Host, error) { host := &kolide.Host{} err := d.db.Get(host, sqlStatement, id) if err != nil { - return nil, errors.Wrap(err, "getting host by id") + return nil, errors.Wrap(err, "get host by id") + } + if err := d.loadHostPackStats(host); err != nil { + return nil, err } return host, nil @@ -605,5 +727,9 @@ func (d *Datastore) HostByIdentifier(identifier string) (*kolide.Host, error) { return nil, errors.Wrap(err, "get host by identifier") } + if err := d.loadHostPackStats(host); err != nil { + return nil, err + } + return host, nil } diff --git a/server/datastore/mysql/migrations/tables/20210506095025_AddScheduledQueryStats.go b/server/datastore/mysql/migrations/tables/20210506095025_AddScheduledQueryStats.go new file mode 100644 index 0000000000..119aa7f9d3 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20210506095025_AddScheduledQueryStats.go @@ -0,0 +1,40 @@ +package tables + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20210506095025, Down_20210506095025) +} + +func Up_20210506095025(tx *sql.Tx) error { + sql := ` + CREATE TABLE scheduled_query_stats ( + host_id int unsigned NOT NULL, + scheduled_query_id int unsigned NOT NULL, + average_memory int, + denylisted tinyint(1), + executions int, + schedule_interval int, + last_executed timestamp, + output_size int, + system_time int, + user_time int, + wall_time int, + PRIMARY KEY (host_id, scheduled_query_id), + FOREIGN KEY (host_id) REFERENCES hosts (id), + FOREIGN KEY (scheduled_query_id) REFERENCES scheduled_queries (id) + ) + ` + if _, err := tx.Exec(sql); err != nil { + return errors.Wrap(err, "create scheduled_query_stats") + } + return nil +} + +func Down_20210506095025(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/scheduled_queries.go b/server/datastore/mysql/scheduled_queries.go index 151c862ecd..0bdbaee3fb 100644 --- a/server/datastore/mysql/scheduled_queries.go +++ b/server/datastore/mysql/scheduled_queries.go @@ -87,7 +87,7 @@ func (d *Datastore) NewScheduledQuery(sq *kolide.ScheduledQuery, opts ...kolide. } sq.Query = metadata[0].Query - sq.Name = metadata[0].Name + sq.QueryName = metadata[0].Name return sq, nil } diff --git a/server/kolide/hosts.go b/server/kolide/hosts.go index 2ac1498d23..6877b2469d 100644 --- a/server/kolide/hosts.go +++ b/server/kolide/hosts.go @@ -145,6 +145,9 @@ type Host struct { LoggerTLSPeriod uint `json:"logger_tls_period" db:"logger_tls_period"` Additional *json.RawMessage `json:"additional,omitempty" db:"additional"` EnrollSecretName string `json:"enroll_secret_name" db:"enroll_secret_name"` + + // Loaded via JOIN in DB + PackStats []PackStats `json:"pack_stats"` } // HostDetail provides the full host metadata along with associated labels and diff --git a/server/kolide/packs.go b/server/kolide/packs.go index 88475eee7d..c614fb4928 100644 --- a/server/kolide/packs.go +++ b/server/kolide/packs.go @@ -176,3 +176,9 @@ type PackTarget struct { PackID uint Target } + +type PackStats struct { + PackID uint `json:"pack_id,omitempty"` + PackName string `json:"pack_name,omitempty"` + QueryStats []ScheduledQueryStats `json:"query_stats"` +} diff --git a/server/kolide/scheduled_queries.go b/server/kolide/scheduled_queries.go index e20dd9af0b..c2f8084128 100644 --- a/server/kolide/scheduled_queries.go +++ b/server/kolide/scheduled_queries.go @@ -2,6 +2,7 @@ package kolide import ( "context" + "time" "gopkg.in/guregu/null.v3" ) @@ -51,3 +52,25 @@ type ScheduledQueryPayload struct { Shard *null.Int `json:"shard"` Denylist *bool `json:"denylist"` } + +type ScheduledQueryStats struct { + ScheduledQueryName string `json:"scheduled_query_name,omitempty" db:"scheduled_query_name"` + ScheduledQueryID uint `json:"scheduled_query_id,omitempty" db:"scheduled_query_id"` + + QueryName string `json:"query_name,omitempty" db:"query_name"` + + PackName string `json:"pack_name,omitempty" db:"pack_name"` + PackID uint `json:"pack_id,omitempty" db:"pack_id"` + + // From osquery directly + AverageMemory int `json:"average_memory" db:"average_memory"` + Denylisted bool `json:"denylisted" db:"denylisted"` + Executions int `json:"executions" db:"executions"` + // Note schedule_interval is used for DB since "interval" is a reserved word in MySQL + Interval int `json:"interval" db:"schedule_interval"` + LastExecuted time.Time `json:"last_executed" db:"last_executed"` + OutputSize int `json:"output_size" db:"output_size"` + SystemTime int `json:"system_time" db:"system_time"` + UserTime int `json:"user_time" db:"user_time"` + WallTime int `json:"wall_time" db:"wall_time"` +} diff --git a/server/service/service_osquery.go b/server/service/service_osquery.go index 8cc1aee194..5b9893570c 100644 --- a/server/service/service_osquery.go +++ b/server/service/service_osquery.go @@ -747,6 +747,80 @@ FROM python_packages; Platforms: []string{"windows"}, IngestFunc: ingestSoftware, }, + "scheduled_query_stats": { + Query: ` + SELECT *, + (SELECT value from osquery_flags where name = 'pack_delimiter') AS delimiter + FROM osquery_schedule +`, + IngestFunc: func(logger log.Logger, host *kolide.Host, rows []map[string]string) error { + packs := map[string][]kolide.ScheduledQueryStats{} + + for _, row := range rows { + providedName := row["name"] + if providedName == "" { + level.Debug(logger).Log( + "msg", "host reported scheduled query with empty name", + "host", host.HostName, + ) + continue + } + delimiter := row["delimiter"] + if delimiter == "" { + level.Debug(logger).Log( + "msg", "host reported scheduled query with empty delimiter", + "host", host.HostName, + ) + continue + } + + // Split with a limit of 2 in case query name includes the + // delimiter. Not much we can do if pack name includes the + // delimiter. + trimmedName := strings.TrimPrefix(providedName, "pack"+delimiter) + parts := strings.SplitN(trimmedName, delimiter, 2) + if len(parts) != 2 { + level.Debug(logger).Log( + "msg", "could not split pack and query names", + "host", host.HostName, + "name", providedName, + "delimiter", delimiter, + ) + continue + } + packName, scheduledName := parts[0], parts[1] + + stats := kolide.ScheduledQueryStats{ + ScheduledQueryName: scheduledName, + PackName: packName, + AverageMemory: cast.ToInt(row["average_memory"]), + Denylisted: cast.ToBool(row["denylisted"]), + Executions: cast.ToInt(row["executions"]), + Interval: cast.ToInt(row["interval"]), + // Cast to int first to allow cast.ToTime to interpret the unix timestamp. + LastExecuted: time.Unix(cast.ToInt64(row["last_executed"]), 0).UTC(), + OutputSize: cast.ToInt(row["output_size"]), + SystemTime: cast.ToInt(row["system_time"]), + UserTime: cast.ToInt(row["user_time"]), + WallTime: cast.ToInt(row["wall_time"]), + } + packs[packName] = append(packs[packName], stats) + } + + host.PackStats = []kolide.PackStats{} + for packName, stats := range packs { + host.PackStats = append( + host.PackStats, + kolide.PackStats{ + PackName: packName, + QueryStats: stats, + }, + ) + } + + return nil + }, + }, } func ingestSoftware(logger log.Logger, host *kolide.Host, rows []map[string]string) error { diff --git a/server/service/service_osquery_test.go b/server/service/service_osquery_test.go index 072b83b42b..23103676d7 100644 --- a/server/service/service_osquery_test.go +++ b/server/service/service_osquery_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "sort" "strconv" "strings" "sync" @@ -971,6 +972,177 @@ func TestDetailQueryNetworkInterfaces(t *testing.T) { assert.Equal(t, "00:00:00:00:00:00", host.PrimaryMac) } +func TestDetailQueryScheduledQueryStats(t *testing.T) { + host := kolide.Host{} + + ingest := detailQueries["scheduled_query_stats"].IngestFunc + + assert.NoError(t, ingest(log.NewNopLogger(), &host, nil)) + assert.Len(t, host.PackStats, 0) + + resJSON := ` +[ + { + "average_memory":"33", + "delimiter":"/", + "denylisted":"0", + "executions":"1", + "interval":"33", + "last_executed":"1620325191", + "name":"pack/pack-2/time", + "output_size":"", + "query":"SELECT * FROM time", + "system_time":"100", + "user_time":"60", + "wall_time":"180" + }, + { + "average_memory":"8000", + "delimiter":"/", + "denylisted":"0", + "executions":"164", + "interval":"30", + "last_executed":"1620325191", + "name":"pack/test/osquery info", + "output_size":"1337", + "query":"SELECT * FROM osquery_info", + "system_time":"150", + "user_time":"180", + "wall_time":"0" + }, + { + "average_memory":"50400", + "delimiter":"/", + "denylisted":"1", + "executions":"188", + "interval":"30", + "last_executed":"1620325203", + "name":"pack/test/processes?", + "output_size":"", + "query":"SELECT * FROM processes", + "system_time":"140", + "user_time":"190", + "wall_time":"1" + }, + { + "average_memory":"0", + "delimiter":"/", + "denylisted":"0", + "executions":"1", + "interval":"3600", + "last_executed":"1620323381", + "name":"pack/test/processes?-1", + "output_size":"", + "query":"SELECT * FROM processes", + "system_time":"0", + "user_time":"0", + "wall_time":"0" + }, + { + "average_memory":"0", + "delimiter":"/", + "denylisted":"0", + "executions":"105", + "interval":"47", + "last_executed":"1620325190", + "name":"pack/test/time", + "output_size":"", + "query":"SELECT * FROM time", + "system_time":"70", + "user_time":"50", + "wall_time":"1" + } +] +` + + var rows []map[string]string + require.NoError(t, json.Unmarshal([]byte(resJSON), &rows)) + + assert.NoError(t, ingest(log.NewNopLogger(), &host, rows)) + assert.Len(t, host.PackStats, 2) + sort.Slice(host.PackStats, func(i, j int) bool { + return host.PackStats[i].PackName < host.PackStats[j].PackName + }) + assert.Equal(t, host.PackStats[0].PackName, "pack-2") + assert.ElementsMatch(t, host.PackStats[0].QueryStats, + []kolide.ScheduledQueryStats{ + { + ScheduledQueryName: "time", + PackName: "pack-2", + AverageMemory: 33, + Denylisted: false, + Executions: 1, + Interval: 33, + LastExecuted: time.Unix(1620325191, 0).UTC(), + OutputSize: 0, + SystemTime: 100, + UserTime: 60, + WallTime: 180, + }, + }, + ) + assert.Equal(t, host.PackStats[1].PackName, "test") + assert.ElementsMatch(t, host.PackStats[1].QueryStats, + []kolide.ScheduledQueryStats{ + { + ScheduledQueryName: "osquery info", + PackName: "test", + AverageMemory: 8000, + Denylisted: false, + Executions: 164, + Interval: 30, + LastExecuted: time.Unix(1620325191, 0).UTC(), + OutputSize: 1337, + SystemTime: 150, + UserTime: 180, + WallTime: 0, + }, + { + ScheduledQueryName: "processes?", + PackName: "test", + AverageMemory: 50400, + Denylisted: true, + Executions: 188, + Interval: 30, + LastExecuted: time.Unix(1620325203, 0).UTC(), + OutputSize: 0, + SystemTime: 140, + UserTime: 190, + WallTime: 1, + }, + { + ScheduledQueryName: "processes?-1", + PackName: "test", + AverageMemory: 0, + Denylisted: false, + Executions: 1, + Interval: 3600, + LastExecuted: time.Unix(1620323381, 0).UTC(), + OutputSize: 0, + SystemTime: 0, + UserTime: 0, + WallTime: 0, + }, + { + ScheduledQueryName: "time", + PackName: "test", + AverageMemory: 0, + Denylisted: false, + Executions: 105, + Interval: 47, + LastExecuted: time.Unix(1620325190, 0).UTC(), + OutputSize: 0, + SystemTime: 70, + UserTime: 50, + WallTime: 1, + }, + }, + ) + + assert.NoError(t, ingest(log.NewNopLogger(), &host, nil)) + assert.Len(t, host.PackStats, 0) +} + func TestNewDistributedQueryCampaign(t *testing.T) { ds := &mock.Store{ AppConfigStore: mock.AppConfigStore{ diff --git a/server/test/new_objects.go b/server/test/new_objects.go index 44124ecc1a..c78ac7220d 100644 --- a/server/test/new_objects.go +++ b/server/test/new_objects.go @@ -9,10 +9,14 @@ import ( ) func NewQuery(t *testing.T, ds kolide.Datastore, name, q string, authorID uint, saved bool) *kolide.Query { + authorPtr := &authorID + if authorID == 0 { + authorPtr = nil + } query, err := ds.NewQuery(&kolide.Query{ Name: name, Query: q, - AuthorID: &authorID, + AuthorID: authorPtr, Saved: saved, }) require.Nil(t, err) @@ -122,8 +126,9 @@ func NewUser(t *testing.T, ds kolide.Datastore, name, username, email string, ad return u } -func NewScheduledQuery(t *testing.T, ds kolide.Datastore, pid, qid, interval uint, snapshot, removed bool) *kolide.ScheduledQuery { +func NewScheduledQuery(t *testing.T, ds kolide.Datastore, pid, qid, interval uint, snapshot, removed bool, name string) *kolide.ScheduledQuery { sq, err := ds.NewScheduledQuery(&kolide.ScheduledQuery{ + Name: name, PackID: pid, QueryID: qid, Interval: interval, From eaf576fda2165c88f14214e01baa347094e786e8 Mon Sep 17 00:00:00 2001 From: Mike Thomas <78363703+mike-j-thomas@users.noreply.github.com> Date: Fri, 7 May 2021 21:22:03 +0900 Subject: [PATCH 13/51] query-library-get-system-uptime (#736) --- handbook/queries/README.md | 1 + handbook/queries/get-system-uptime.md | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 handbook/queries/get-system-uptime.md diff --git a/handbook/queries/README.md b/handbook/queries/README.md index 5ee5c5b46d..31682612b9 100644 --- a/handbook/queries/README.md +++ b/handbook/queries/README.md @@ -27,6 +27,7 @@ Fleet's standard query library includes a growing collection of useful queries f - [Get OS version](./get-os-version.md) (macOS, Linux, Windows, FreeBSD) - [Get mounts](./get-mounts.md) (macOS, Linux) - [Get startup items](./get-startup-items.md) (macOS, Linux, Windows, FreeBSD) +- [Get system uptime](./get-system-uptime.md) (macOS, Linux, Windows, FreeBSD) ### Contributors diff --git a/handbook/queries/get-system-uptime.md b/handbook/queries/get-system-uptime.md new file mode 100644 index 0000000000..ccc42b24c0 --- /dev/null +++ b/handbook/queries/get-system-uptime.md @@ -0,0 +1,16 @@ +# Get system uptime + +Shows the system uptime. + +### Support +macOS, Linux, Windows, FreeBSD + +### Query +```sql +SELECT * FROM uptime; +``` +### Purpose +Informational + +### Remediation +N/A \ No newline at end of file From 5af0d6b8708ab637a1a5e3016a5726353878de9e Mon Sep 17 00:00:00 2001 From: gillespi314 <73313222+gillespi314@users.noreply.github.com> Date: Fri, 7 May 2021 13:54:29 -0500 Subject: [PATCH 14/51] Update website styling (#715) * Update website styling * Convert ellipse margin from rem to px * Adjust padding to match neighboring card element * Fix linting errors * Remove unneeded div --- .../assets/styles/bootstrap-overrides.less | 21 +- website/assets/styles/layout.less | 40 ++++ website/assets/styles/pages/get-started.less | 28 +-- website/assets/styles/pages/homepage.less | 3 - website/assets/styles/pages/pricing.less | 30 +-- website/views/layouts/layout.ejs | 14 +- website/views/pages/get-started.ejs | 2 +- website/views/pages/homepage.ejs | 126 ++++++------ website/views/pages/pricing.ejs | 179 +++++++----------- 9 files changed, 225 insertions(+), 218 deletions(-) diff --git a/website/assets/styles/bootstrap-overrides.less b/website/assets/styles/bootstrap-overrides.less index 5ff72e033f..f5c0f36a76 100644 --- a/website/assets/styles/bootstrap-overrides.less +++ b/website/assets/styles/bootstrap-overrides.less @@ -51,7 +51,6 @@ footer { line-height: 24px; } - .btn { font-family: @header-font; font-weight: 700; @@ -79,25 +78,25 @@ footer { } &.btn-primary { - background-color: #EF7182; - border-color: #EF7182; + background-color: #ff5c83; + border-color: #ff5c83; &:hover { - background-color: darken(#EF7182, 10%); - border-color: darken(#EF7182, 10%); + background-color: darken(#ff5c83, 10%); + border-color: darken(#ff5c83, 10%); } } &.btn-info { - background-color: #6B6CFA; - border-color: #6B6CFA; + background-color: #6b6cfa; + border-color: #6b6cfa; &:hover { - background-color: darken(#6B6CFA, 10%); - border-color: darken(#6B6CFA, 10%); + background-color: darken(#6b6cfa, 10%); + border-color: darken(#6b6cfa, 10%); } } } .card { - box-shadow: 0px 4px 10px rgba(0, 0, 0, 0.1); + box-shadow: 0px 6px 20px rgba(0, 0, 0, 0.05); } .dropdown:hover > .dropdown-menu { @@ -108,7 +107,7 @@ footer { } .dropdown-menu { - left: 16px !important;//lesshint-disable-line importantRule + left: 16px !important; //lesshint-disable-line importantRule min-width: 0; margin: 0; } diff --git a/website/assets/styles/layout.less b/website/assets/styles/layout.less index d013930be7..005aa1cc77 100644 --- a/website/assets/styles/layout.less +++ b/website/assets/styles/layout.less @@ -80,6 +80,24 @@ html, body { background-color: #f1f0ff; font-weight: @bold; } + .menu-link { + color: #192147; + font-weight: @bold; + svg { + path { + fill: #192147; + } + } + } + .menu-link:hover { + color: #6a67fe; + + svg { + path { + fill: #6a67fe; + } + } + } } [purpose='page-header'].header { @@ -141,6 +159,24 @@ html, body { background-color: #f1f0ff; font-weight: @bold; } + .menu-link { + color: #192147; + font-weight: @bold; + svg { + path { + fill: #192147; + } + } + } + .menu-link:hover { + color: #6a67fe; + + svg { + path { + fill: #6a67fe; + } + } + } } [purpose='page-footer'] { @@ -177,6 +213,10 @@ body.detected-mobile { // … } +.dropdown:hover > .btn.btn-link { + color: #6a67fe; +} + @media (max-width: 1199px) { [purpose='page-wrap'] { padding-bottom: 230px; diff --git a/website/assets/styles/pages/get-started.less b/website/assets/styles/pages/get-started.less index d52f62c6dd..73bb9fcdea 100644 --- a/website/assets/styles/pages/get-started.less +++ b/website/assets/styles/pages/get-started.less @@ -9,7 +9,7 @@ color: #6a67fe; } #primary-button { - color: #fff; + color: #fff; } #learn-more { background: #f9fafc; @@ -33,6 +33,7 @@ border-color: #f9fafc; } } + #arrow { padding-right: 45px; cursor: pointer; @@ -41,11 +42,11 @@ color: #192147; font-weight: bold; user-select: none; - transition: 0.2s linear; - -o-transition: 0.2s linear; - -ms-transition: 0.2s linear; - -moz-transition: 0.2s linear; - -webkit-transition: 0.2s linear; + transition: 0.2s ease-in-out; + -o-transition: 0.2s ease-in-out; + -ms-transition: 0.2s ease-in-out; + -moz-transition: 0.2s ease-in-out; + -webkit-transition: 0.2s ease-in-out; } #arrow:before { @@ -54,16 +55,21 @@ position: absolute; top: 12px; left: 80%; // <--- here + transition: 0.2s ease-in-out; + -o-transition: 0.2s ease-in-out; + -ms-transition: 0.2s ease-in-out; + -moz-transition: 0.2s ease-in-out; + -webkit-transition: 0.2s ease-in-out; /* opacity: 0; */ } #arrow:hover:before { left: 82%; // <--- here - transition: 0.2s linear; - -o-transition: 0.2s linear; - -ms-transition: 0.2s linear; - -moz-transition: 0.2s linear; - -webkit-transition: 0.2s linear; + transition: 0.2s ease-in-out; + -o-transition: 0.2s ease-in-out; + -ms-transition: 0.2s ease-in-out; + -moz-transition: 0.2s ease-in-out; + -webkit-transition: 0.2s ease-in-out; /* opacity:1; */ } } diff --git a/website/assets/styles/pages/homepage.less b/website/assets/styles/pages/homepage.less index e7bad929d3..91867093cf 100644 --- a/website/assets/styles/pages/homepage.less +++ b/website/assets/styles/pages/homepage.less @@ -27,9 +27,6 @@ .quote-mark { font-family: @header-font; } - .footer-fade { - background: linear-gradient(360deg, #201E43 0%, #353D62 100%); - } .log-destinations-container { display: flex; flex-wrap: wrap; diff --git a/website/assets/styles/pages/pricing.less b/website/assets/styles/pages/pricing.less index d58de5e90e..6adfce6e85 100644 --- a/website/assets/styles/pages/pricing.less +++ b/website/assets/styles/pages/pricing.less @@ -21,11 +21,6 @@ border-color: #fff; } } - .card-body { - box-shadow: 0px 6px 20px rgba(0, 0, 0, 0.05); - border: 1px solid #e2e4ea; - border-radius: 0.75rem; - } #arrow { padding-right: 45px; @@ -35,11 +30,11 @@ color: #192147; font-weight: bold; user-select: none; - transition: 0.2s linear; - -o-transition: 0.2s linear; - -ms-transition: 0.2s linear; - -moz-transition: 0.2s linear; - -webkit-transition: 0.2s linear; + transition: 0.2s ease-in-out; + -o-transition: 0.2s ease-in-out; + -ms-transition: 0.2s ease-in-out; + -moz-transition: 0.2s ease-in-out; + -webkit-transition: 0.2s ease-in-out; } #arrow:before { @@ -48,16 +43,21 @@ position: absolute; top: 12px; left: 80%; // <--- here + transition: 0.2s ease-in-out; + -o-transition: 0.2s ease-in-out; + -ms-transition: 0.2s ease-in-out; + -moz-transition: 0.2s ease-in-out; + -webkit-transition: 0.2s ease-in-out; /* opacity: 0; */ } #arrow:hover:before { left: 82%; // <--- here - transition: 0.2s linear; - -o-transition: 0.2s linear; - -ms-transition: 0.2s linear; - -moz-transition: 0.2s linear; - -webkit-transition: 0.2s linear; + transition: 0.2s ease-in-out; + -o-transition: 0.2s ease-in-out; + -ms-transition: 0.2s ease-in-out; + -moz-transition: 0.2s ease-in-out; + -webkit-transition: 0.2s ease-in-out; /* opacity:1; */ } } diff --git a/website/views/layouts/layout.ejs b/website/views/layouts/layout.ejs index a6e9da0944..7a05335eb1 100644 --- a/website/views/layouts/layout.ejs +++ b/website/views/layouts/layout.ejs @@ -86,11 +86,13 @@ Contribute Hall of fame
- Pricing - Blog - - - A blue octocat icon representing the GitHub brand. + Pricing + Blog + + + + + GitHub
@@ -98,7 +100,7 @@