Discharge7443/fleet
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

2023-09-16: Website: Update sitemap, set lastModifedAt timestamps for osquery schema pages. (#13725)

Browse Source 
Closes: #13728

Changes:
- Added /support to the array of hand-coded HTML pages in
`download-sitemap.js`:
- Updated `get-extended-osquery-schema`:
- Added a new (optional) input: `includeLastModifiedAtValue` if this
input is provided, the helper will:
      - Set a `lastModifiedAt` value on all tables.
- Send a request to the GitHub API to get a lastModifiedAt timestamp for
tables that have no fleet overrides.
- Use `git` to get a lastModifiedAt timestamp of when the tables YAML
file was changed.
- Updated the `build-static-content` script to include a lastModifiedAt
timestamp for table pages, and updated the `lastModifiedAt` value that
is set for pages built from `/handbook/company/open-positions.yml`

---------

Co-authored-by: Mike McNeil <mikermcneil@users.noreply.github.com>
This commit is contained in:
Eric
2023-09-15 18:39:39 -05:00
committed by GitHub
co-authored by Mike McNeil
parent f894726d73
commit 85fdeb4234
3 changed files with 65 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
website/api/controllers/download-sitemap.js
Vendored
+1 -1
View File
@@ -52,7 +52,6 @@ module.exports = {
'/docs',
'/logos',
'/reports/state-of-device-management',
'/overview',
'/releases',
'/success-stories',
'/securing',
@@ -63,6 +62,7 @@ module.exports = {
'/deploy',
'/podcasts',
'/device-management',
'/support',
// FUTURE: Do something smarter to get hand-coded HTML pages from routes.js, like how rebuild-cloud-sdk works, to avoid this manual duplication.
// See also https://github.com/sailshq/sailsjs.com/blob/b53c6e6a90c9afdf89e5cae00b9c9dd3f391b0e7/api/helpers/get-pages-for-sitemap.js#L27
];
website/api/helpers/get-extended-osquery-schema.js
Vendored
+53 -2
View File
@@ -6,6 +6,13 @@ module.exports = {
description: 'Get the extended osquery schema and documentation supported by Fleet by reading the raw osquery tables and Fleet\'s overrides from disk, then returning the extended set of tables.',
inputs: {
includeLastModifiedAtValue: {
type: 'boolean',
defaultsTo: false,
description: 'Whether or not to include a lastModifiedAt value for each table.',
}
},
exits: {
@@ -18,9 +25,11 @@ module.exports = {
},
fn: async function () {
fn: async function ({includeLastModifiedAtValue}) {
let path = require('path');
let YAML = require('yaml');
let util = require('util');
let topLvlRepoPath = path.resolve(sails.config.appPath, '../');
require('assert')(sails.config.custom.versionOfOsquerySchemaToUseWhenGeneratingDocumentation, 'Please set sails.config.custom.sails.config.custom.versionOfOsquerySchemaToUseWhenGeneratingDocumentation to the version of osquery to use, for example \'5.8.1\'.');
@@ -29,6 +38,31 @@ module.exports = {
// Getting the specified osquery schema from the osquery/osquery-site GitHub repo.
let rawOsqueryTables = await sails.helpers.http.get('https://raw.githubusercontent.com/osquery/osquery-site/source/src/data/osquery_schema_versions/'+VERSION_OF_OSQUERY_SCHEMA_TO_USE+'.json');
let rawOsqueryTablesLastModifiedAt;
if(includeLastModifiedAtValue) {
// If we're including a lastModifiedAt value for schema tables, we'll send a request to the GitHub API to get a timestamp of when the last commit
let responseData = await sails.helpers.http.get.with({// [?]: https://docs.github.com/en/rest/commits/commits?apiVersion=2022-11-28#list-commits
url: 'https://api.github.com/repos/osquery/osquery-site/commits',
data: {
path: '/src/data/osquery_schema_versions/'+VERSION_OF_OSQUERY_SCHEMA_TO_USE+'.json',
page: 1,
per_page: 1,//eslint-disable-line camelcase
},
headers: {
'User-Agent': 'fleet-schema-builder',
'Accept': 'application/vnd.github.v3+json',
},
}).intercept((err)=>{
return new Error(`When trying to send a request to GitHub get a timestamp of the last commit to the osqeury schema JSON, an error occurred. Full error: ${util.inspect(err)}`);
});
// The value we'll use for the lastModifiedAt timestamp will be date value of the `commiter` property of the `commit` we got in the API response from github.
let mostRecentCommitToOsquerySchema = responseData[0];
if(!mostRecentCommitToOsquerySchema.commit || !mostRecentCommitToOsquerySchema.commit.committer) {
// Throw an error if the the response from GitHub is missing a commit or commiter.
throw new Error(`When trying to get a lastModifiedAt timestamp for the osqeury schema json, the response from the GitHub API did not include information about the most recent commit. Response from GitHub: ${util.inspect(responseData, {depth:null})}`);
}
rawOsqueryTablesLastModifiedAt = (new Date(mostRecentCommitToOsquerySchema.commit.committer.date)).getTime(); // Convert the UTC timestamp from GitHub to a JS timestamp.
}
let fleetOverridesForTables = [];
let filesInTablesFolder = await sails.helpers.fs.ls(path.resolve(topLvlRepoPath+'/schema/tables'));
@@ -43,6 +77,17 @@ module.exports = {
} catch(err) {
throw new Error(`Could not parse the Fleet overrides YAMl at ${yamlSchema} on line ${err.linePos.start.line}. To resolve, make sure the YAML is valid, then try running this script again: `+err.stack);
}
if(includeLastModifiedAtValue) {
// If we're including lastModifiedAt values, we'll use git to get a timestamp representing when the yaml
// file was last changed, and add it to the parsedYamlTable object.
let lastModifiedAt = (new Date((await sails.helpers.process.executeCommand.with({
command: `git log -1 --format="%ai" '${path.relative(topLvlRepoPath, yamlSchema)}'`,
dir: topLvlRepoPath,
})).stdout)).getTime();
parsedYamlTable.lastModifiedAt = lastModifiedAt;
}
if(parsedYamlTable.name) {
if(typeof parsedYamlTable.name !== 'string') {
throw new Error(`Could not merge osquery schema with Fleet overrides. A table in the Fleet overrides schema has an invalid "name" (Expected a string, but instead got a ${typeof parsedYamlTable.name}. To resolve, change the "name" of the table located at ${yamlSchema} to be a string.`);
@@ -89,7 +134,9 @@ module.exports = {
// fence so it renders as a code block.
expandedTableToPush.examples = '```\n' + examplesFromOsquerySchema[examplesFromOsquerySchema.length - 1] + '\n```';
}
if(includeLastModifiedAtValue) {
expandedTableToPush.lastModifiedAt = rawOsqueryTablesLastModifiedAt;
}
expandedTables.push(expandedTableToPush);
} else { // If this table exists in the Fleet overrides schema, we'll override the values
if(fleetOverridesForTable.platforms !== undefined) {
@@ -130,6 +177,10 @@ module.exports = {
// If the table has Fleet overrides, we'll add the URL of the YAML file in the Fleet Github repo as the `fleetRepoUrl`, and add set the url to be where this table will live on fleetdm.com.
expandedTableToPush.fleetRepoUrl = 'https://github.com/fleetdm/fleet/blob/main/schema/tables/'+encodeURIComponent(expandedTableToPush.name)+'.yml';
expandedTableToPush.url = 'https://fleetdm.com/tables/'+encodeURIComponent(expandedTableToPush.name);
// If we're including lastModifiedAt values, we'll set the value for this table to be when the Fleet override was last modified.
if(includeLastModifiedAtValue) {
expandedTableToPush.lastModifiedAt = fleetOverridesForTable.lastModifiedAt;
}
let mergedTableColumns = [];
for (let osquerySchemaColumn of osquerySchemaTable.columns) { // iterate through the columns in the osquery schema table
if(!fleetOverridesForTable.columns) { // If there are no column overrides for this table, we'll add the column unchanged.
website/scripts/build-static-content.js
Vendored
+11 -2
View File
@@ -566,6 +566,14 @@ module.exports = {
// Now build EJS partials from open positions in open-positions.yml. Note: We don't build these
builtStaticContent.openPositions = [];// This will be passed into a component on the company handbook page to render a list of open positions.
let RELATIVE_PATH_TO_OPEN_POSITIONS_YML_IN_FLEET_REPO = 'handbook/company/open-positions.yml';
// Get last modified timestamp using git, and represent it as a JS timestamp.
// > Inspired by https://github.com/uncletammy/doc-templater/blob/2969726b598b39aa78648c5379e4d9503b65685e/lib/compile-markdown-tree-from-remote-git-repo.js#L265-L273
let lastModifiedAt = (new Date((await sails.helpers.process.executeCommand.with({
command: `git log -1 --format="%ai" '${path.join(topLvlRepoPath, RELATIVE_PATH_TO_OPEN_POSITIONS_YML_IN_FLEET_REPO)}'`,
dir: topLvlRepoPath,
})).stdout)).getTime();
let openPositionsYaml = await sails.helpers.fs.read(path.join(topLvlRepoPath, RELATIVE_PATH_TO_OPEN_POSITIONS_YML_IN_FLEET_REPO)).intercept('doesNotExist', (err)=>new Error(`Could not find open positions YAML file at "${RELATIVE_PATH_TO_OPEN_POSITIONS_YML_IN_FLEET_REPO}". Was it accidentally moved? Raw error: `+err.message));
let openPositionsToCreatePartialsFor = YAML.parse(openPositionsYaml, {prettyErrors: true});
@@ -658,7 +666,7 @@ module.exports = {
builtStaticContent.markdownPages.push({
url: rootRelativeUrlPath,
title: pageTitle,
lastModifiedAt: Date.now(),
lastModifiedAt: lastModifiedAt,
htmlId: htmlId,
sectionRelativeRepoPath: 'company/open-positions.yml', // This is used to create the url for the "Edit this page" link
meta: {maintainedBy: openPosition.hiringManagerGithubUsername},// Set the page maintainer to be the position's hiring manager.
@@ -671,7 +679,7 @@ module.exports = {
}
// After we build the Markdown pages, we'll merge the osquery schema with the Fleet schema overrides, then create EJS partials for each table in the merged schema.
let expandedTables = await sails.helpers.getExtendedOsquerySchema();
let expandedTables = await sails.helpers.getExtendedOsquerySchema.with({includeLastModifiedAtValue: true});
// Once we have our merged schema, we'll create ejs partials for each table.
for(let table of expandedTables) {
@@ -782,6 +790,7 @@ module.exports = {
title: table.name,
htmlId: htmlId,
evented: table.evented,
lastModifiedAt: table.lastModifiedAt,
platforms: table.platforms,
keywordsForSyntaxHighlighting: keywordsForSyntaxHighlighting,
sectionRelativeRepoPath: table.name, // Setting the sectionRelativeRepoPath to an arbitrary string to work with existing pages.