From 85fdeb423442519abe4c44236b98b2ee644341d2 Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 15 Sep 2023 18:39:39 -0500 Subject: [PATCH] 2023-09-16: Website: Update sitemap, set `lastModifedAt` timestamps for osquery schema pages. (#13725) Closes: #13728 Changes: - Added /support to the array of hand-coded HTML pages in `download-sitemap.js`: - Updated `get-extended-osquery-schema`: - Added a new (optional) input: `includeLastModifiedAtValue` if this input is provided, the helper will: - Set a `lastModifiedAt` value on all tables. - Send a request to the GitHub API to get a lastModifiedAt timestamp for tables that have no fleet overrides. - Use `git` to get a lastModifiedAt timestamp of when the tables YAML file was changed. - Updated the `build-static-content` script to include a lastModifiedAt timestamp for table pages, and updated the `lastModifiedAt` value that is set for pages built from `/handbook/company/open-positions.yml` --------- Co-authored-by: Mike McNeil --- website/api/controllers/download-sitemap.js | 2 +- .../helpers/get-extended-osquery-schema.js | 55 ++++++++++++++++++- website/scripts/build-static-content.js | 13 ++++- 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/website/api/controllers/download-sitemap.js b/website/api/controllers/download-sitemap.js index 4e6ae26c39..483a901693 100644 --- a/website/api/controllers/download-sitemap.js +++ b/website/api/controllers/download-sitemap.js @@ -52,7 +52,6 @@ module.exports = { '/docs', '/logos', '/reports/state-of-device-management', - '/overview', '/releases', '/success-stories', '/securing', @@ -63,6 +62,7 @@ module.exports = { '/deploy', '/podcasts', '/device-management', + '/support', // FUTURE: Do something smarter to get hand-coded HTML pages from routes.js, like how rebuild-cloud-sdk works, to avoid this manual duplication. // See also https://github.com/sailshq/sailsjs.com/blob/b53c6e6a90c9afdf89e5cae00b9c9dd3f391b0e7/api/helpers/get-pages-for-sitemap.js#L27 ]; diff --git a/website/api/helpers/get-extended-osquery-schema.js b/website/api/helpers/get-extended-osquery-schema.js index 62242db3ee..63f521886f 100644 --- a/website/api/helpers/get-extended-osquery-schema.js +++ b/website/api/helpers/get-extended-osquery-schema.js @@ -6,6 +6,13 @@ module.exports = { description: 'Get the extended osquery schema and documentation supported by Fleet by reading the raw osquery tables and Fleet\'s overrides from disk, then returning the extended set of tables.', + inputs: { + includeLastModifiedAtValue: { + type: 'boolean', + defaultsTo: false, + description: 'Whether or not to include a lastModifiedAt value for each table.', + } + }, exits: { @@ -18,9 +25,11 @@ module.exports = { }, - fn: async function () { + fn: async function ({includeLastModifiedAtValue}) { let path = require('path'); let YAML = require('yaml'); + let util = require('util'); + let topLvlRepoPath = path.resolve(sails.config.appPath, '../'); require('assert')(sails.config.custom.versionOfOsquerySchemaToUseWhenGeneratingDocumentation, 'Please set sails.config.custom.sails.config.custom.versionOfOsquerySchemaToUseWhenGeneratingDocumentation to the version of osquery to use, for example \'5.8.1\'.'); @@ -29,6 +38,31 @@ module.exports = { // Getting the specified osquery schema from the osquery/osquery-site GitHub repo. let rawOsqueryTables = await sails.helpers.http.get('https://raw.githubusercontent.com/osquery/osquery-site/source/src/data/osquery_schema_versions/'+VERSION_OF_OSQUERY_SCHEMA_TO_USE+'.json'); + let rawOsqueryTablesLastModifiedAt; + if(includeLastModifiedAtValue) { + // If we're including a lastModifiedAt value for schema tables, we'll send a request to the GitHub API to get a timestamp of when the last commit + let responseData = await sails.helpers.http.get.with({// [?]: https://docs.github.com/en/rest/commits/commits?apiVersion=2022-11-28#list-commits + url: 'https://api.github.com/repos/osquery/osquery-site/commits', + data: { + path: '/src/data/osquery_schema_versions/'+VERSION_OF_OSQUERY_SCHEMA_TO_USE+'.json', + page: 1, + per_page: 1,//eslint-disable-line camelcase + }, + headers: { + 'User-Agent': 'fleet-schema-builder', + 'Accept': 'application/vnd.github.v3+json', + }, + }).intercept((err)=>{ + return new Error(`When trying to send a request to GitHub get a timestamp of the last commit to the osqeury schema JSON, an error occurred. Full error: ${util.inspect(err)}`); + }); + // The value we'll use for the lastModifiedAt timestamp will be date value of the `commiter` property of the `commit` we got in the API response from github. + let mostRecentCommitToOsquerySchema = responseData[0]; + if(!mostRecentCommitToOsquerySchema.commit || !mostRecentCommitToOsquerySchema.commit.committer) { + // Throw an error if the the response from GitHub is missing a commit or commiter. + throw new Error(`When trying to get a lastModifiedAt timestamp for the osqeury schema json, the response from the GitHub API did not include information about the most recent commit. Response from GitHub: ${util.inspect(responseData, {depth:null})}`); + } + rawOsqueryTablesLastModifiedAt = (new Date(mostRecentCommitToOsquerySchema.commit.committer.date)).getTime(); // Convert the UTC timestamp from GitHub to a JS timestamp. + } let fleetOverridesForTables = []; let filesInTablesFolder = await sails.helpers.fs.ls(path.resolve(topLvlRepoPath+'/schema/tables')); @@ -43,6 +77,17 @@ module.exports = { } catch(err) { throw new Error(`Could not parse the Fleet overrides YAMl at ${yamlSchema} on line ${err.linePos.start.line}. To resolve, make sure the YAML is valid, then try running this script again: `+err.stack); } + + if(includeLastModifiedAtValue) { + // If we're including lastModifiedAt values, we'll use git to get a timestamp representing when the yaml + // file was last changed, and add it to the parsedYamlTable object. + let lastModifiedAt = (new Date((await sails.helpers.process.executeCommand.with({ + command: `git log -1 --format="%ai" '${path.relative(topLvlRepoPath, yamlSchema)}'`, + dir: topLvlRepoPath, + })).stdout)).getTime(); + parsedYamlTable.lastModifiedAt = lastModifiedAt; + } + if(parsedYamlTable.name) { if(typeof parsedYamlTable.name !== 'string') { throw new Error(`Could not merge osquery schema with Fleet overrides. A table in the Fleet overrides schema has an invalid "name" (Expected a string, but instead got a ${typeof parsedYamlTable.name}. To resolve, change the "name" of the table located at ${yamlSchema} to be a string.`); @@ -89,7 +134,9 @@ module.exports = { // fence so it renders as a code block. expandedTableToPush.examples = '```\n' + examplesFromOsquerySchema[examplesFromOsquerySchema.length - 1] + '\n```'; } - + if(includeLastModifiedAtValue) { + expandedTableToPush.lastModifiedAt = rawOsqueryTablesLastModifiedAt; + } expandedTables.push(expandedTableToPush); } else { // If this table exists in the Fleet overrides schema, we'll override the values if(fleetOverridesForTable.platforms !== undefined) { @@ -130,6 +177,10 @@ module.exports = { // If the table has Fleet overrides, we'll add the URL of the YAML file in the Fleet Github repo as the `fleetRepoUrl`, and add set the url to be where this table will live on fleetdm.com. expandedTableToPush.fleetRepoUrl = 'https://github.com/fleetdm/fleet/blob/main/schema/tables/'+encodeURIComponent(expandedTableToPush.name)+'.yml'; expandedTableToPush.url = 'https://fleetdm.com/tables/'+encodeURIComponent(expandedTableToPush.name); + // If we're including lastModifiedAt values, we'll set the value for this table to be when the Fleet override was last modified. + if(includeLastModifiedAtValue) { + expandedTableToPush.lastModifiedAt = fleetOverridesForTable.lastModifiedAt; + } let mergedTableColumns = []; for (let osquerySchemaColumn of osquerySchemaTable.columns) { // iterate through the columns in the osquery schema table if(!fleetOverridesForTable.columns) { // If there are no column overrides for this table, we'll add the column unchanged. diff --git a/website/scripts/build-static-content.js b/website/scripts/build-static-content.js index 2556e8cd4a..7f0426c2b3 100644 --- a/website/scripts/build-static-content.js +++ b/website/scripts/build-static-content.js @@ -566,6 +566,14 @@ module.exports = { // Now build EJS partials from open positions in open-positions.yml. Note: We don't build these builtStaticContent.openPositions = [];// This will be passed into a component on the company handbook page to render a list of open positions. let RELATIVE_PATH_TO_OPEN_POSITIONS_YML_IN_FLEET_REPO = 'handbook/company/open-positions.yml'; + + // Get last modified timestamp using git, and represent it as a JS timestamp. + // > Inspired by https://github.com/uncletammy/doc-templater/blob/2969726b598b39aa78648c5379e4d9503b65685e/lib/compile-markdown-tree-from-remote-git-repo.js#L265-L273 + let lastModifiedAt = (new Date((await sails.helpers.process.executeCommand.with({ + command: `git log -1 --format="%ai" '${path.join(topLvlRepoPath, RELATIVE_PATH_TO_OPEN_POSITIONS_YML_IN_FLEET_REPO)}'`, + dir: topLvlRepoPath, + })).stdout)).getTime(); + let openPositionsYaml = await sails.helpers.fs.read(path.join(topLvlRepoPath, RELATIVE_PATH_TO_OPEN_POSITIONS_YML_IN_FLEET_REPO)).intercept('doesNotExist', (err)=>new Error(`Could not find open positions YAML file at "${RELATIVE_PATH_TO_OPEN_POSITIONS_YML_IN_FLEET_REPO}". Was it accidentally moved? Raw error: `+err.message)); let openPositionsToCreatePartialsFor = YAML.parse(openPositionsYaml, {prettyErrors: true}); @@ -658,7 +666,7 @@ module.exports = { builtStaticContent.markdownPages.push({ url: rootRelativeUrlPath, title: pageTitle, - lastModifiedAt: Date.now(), + lastModifiedAt: lastModifiedAt, htmlId: htmlId, sectionRelativeRepoPath: 'company/open-positions.yml', // This is used to create the url for the "Edit this page" link meta: {maintainedBy: openPosition.hiringManagerGithubUsername},// Set the page maintainer to be the position's hiring manager. @@ -671,7 +679,7 @@ module.exports = { } // After we build the Markdown pages, we'll merge the osquery schema with the Fleet schema overrides, then create EJS partials for each table in the merged schema. - let expandedTables = await sails.helpers.getExtendedOsquerySchema(); + let expandedTables = await sails.helpers.getExtendedOsquerySchema.with({includeLastModifiedAtValue: true}); // Once we have our merged schema, we'll create ejs partials for each table. for(let table of expandedTables) { @@ -782,6 +790,7 @@ module.exports = { title: table.name, htmlId: htmlId, evented: table.evented, + lastModifiedAt: table.lastModifiedAt, platforms: table.platforms, keywordsForSyntaxHighlighting: keywordsForSyntaxHighlighting, sectionRelativeRepoPath: table.name, // Setting the sectionRelativeRepoPath to an arbitrary string to work with existing pages.