Discharge7443/fleet
1
0
Fork 0
Code Issues Pull Requests Actions 19 Packages Projects Releases Wiki Activity

Speed up CI build/test (#657)

Browse Source 
- Use NODE_ENV=development for Webpack in CI.
- Add caching for Webpack modules in development.
- Remove time of day from dates in version string (helps with build caching).
- Use Go cache in CI.

This makes about a 20% savings on the total time for the CI run.
This commit is contained in:
Zach Wasserman
2021-04-16 16:14:56 -07:00
committed by GitHub
parent 568efc9497
commit 1e78c0ffe2
5 changed files with 188 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/test.yml
+32 -30
View File
@@ -15,21 +15,42 @@ jobs:
uses: actions/checkout@v2
- name: JS Dependency Cache
id: js-cache
uses: actions/cache@v2
with:
path: |
**/node_modules
~/.cache/Cypress
key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
# Use a separate cache for this from other JS jobs since we run the
# webpack steps and will have more to cache.
key: ${{ runner.os }}-e2e-modules-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-modules-
${{ runner.os }}-e2e-modules-
# It seems faster not to cache Go dependencies
- name: Go Cache
id: go-cache
uses: actions/cache@v2
with:
# In order:
# * Module download cache
# * Build cache (Linux)
# * Build cache (Mac)
# * Build cache (Windows)
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Install JS Dependencies
if: steps.js-cache.outputs.cache-hit != 'true'
run: make deps-js
- name: Install Go Dependencies
if: steps.go-cache.outputs.cache-hit != 'true'
run: make deps-go
# Pre-starting dependencies here means they are ready to go when we need them.
@@ -40,7 +61,7 @@ jobs:
- name: Build Fleet
run: |
export PATH=$PATH:~/go/bin
make generate
make generate-ci
make
- name: Run E2E Tests
@@ -63,6 +84,7 @@ jobs:
uses: actions/checkout@v2
- name: JS Dependency Cache
id: js-cache
uses: actions/cache@v2
with:
path: |
@@ -73,6 +95,7 @@ jobs:
${{ runner.os }}-modules-
- name: Install JS Dependencies
if: steps.js-cache.outputs.cache-hit != 'true'
run: make deps-js
- name: Run JS Tests
@@ -91,6 +114,7 @@ jobs:
uses: actions/checkout@v2
- name: JS Dependency Cache
id: js-cache
uses: actions/cache@v2
with:
path: |
@@ -101,38 +125,16 @@ jobs:
${{ runner.os }}-modules-
- name: Install JS Dependencies
if: steps.js-cache.outputs.cache-hit != 'true'
run: make deps-js
- name: Run JS Linting
run: |
make lint-js
check-prettier:
strategy:
matrix:
os: [ ubuntu-latest ]
runs-on: ${{ matrix.os }}
steps:
- name: Checkout Code
uses: actions/checkout@v2
- name: JS Dependency Cache
uses: actions/cache@v2
with:
path: |
**/node_modules
~/.cache/Cypress
key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-modules-
- name: Install JS Dependencies
run: make deps-js
- name: Run prettier formatting check
run: |
yarn prettier:check
- name: Run prettier formatting check
run: |
yarn prettier:check
test-go:
strategy:
Makefile
+6 -2
View File
@@ -21,7 +21,7 @@ ifneq ($(OS), Windows_NT)
# To populate version metadata, we use unix tools to get certain data
GOVERSION = $(shell go version | awk '{print $$3}')
NOW = $(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
NOW = $(shell date +"%Y-%m-%d")
else
# The output binary name is different on Windows, so we're explicit here
OUTPUT = fleet.exe
@@ -29,7 +29,7 @@ else
# To populate version metadata, we use windows tools to get the certain data
GOVERSION_CMD = "(go version).Split()[2]"
GOVERSION = $(shell powershell $(GOVERSION_CMD))
NOW = $(shell powershell Get-Date -format s)
NOW = $(shell powershell Get-Date -format "yyy-MM-dd")
endif
ifndef CIRCLE_PR_NUMBER
@@ -131,6 +131,10 @@ test: lint test-go test-js
generate: clean-assets generate-js generate-go
generate-ci:
NODE_ENV=development webpack
make generate-go
generate-js: clean-assets .prefix
NODE_ENV=production webpack --progress --colors
package.json
+1
View File
@@ -174,6 +174,7 @@
"eslint-plugin-react-hooks": "^4.2.0",
"expose-loader": "0.7.5",
"file-loader": "^4.1.0",
"hard-source-webpack-plugin": "^0.13.1",
"html-webpack-plugin": "^3.2.0",
"identity-obj-proxy": "^3.0.0",
"ignore-styles": "5.0.1",
webpack.config.js
+64 -50
View File
@@ -1,75 +1,92 @@
require('es6-promise').polyfill();
require("es6-promise").polyfill();
const path = require('path');
const webpack = require('webpack');
const bourbon = require('node-bourbon').includePaths;
const HtmlWebpackPlugin = require('html-webpack-plugin');
const MiniCssExtractPlugin = require('mini-css-extract-plugin');
const WebpackNotifierPlugin = require('webpack-notifier');
const path = require("path");
const webpack = require("webpack");
const bourbon = require("node-bourbon").includePaths;
const HtmlWebpackPlugin = require("html-webpack-plugin");
const MiniCssExtractPlugin = require("mini-css-extract-plugin");
const WebpackNotifierPlugin = require("webpack-notifier");
const HardSourceWebpackPlugin = require("hard-source-webpack-plugin");
const DEV_SOURCE_MAPS = 'eval-source-map';
const DEV_SOURCE_MAPS = "eval-source-map";
var plugins = [
new webpack.NoEmitOnErrorsPlugin(),
new HtmlWebpackPlugin({
filename: '../frontend/templates/react.tmpl',
filename: "../frontend/templates/react.tmpl",
inject: false,
template: 'frontend/templates/react.ejs'
template: "frontend/templates/react.ejs",
}),
new WebpackNotifierPlugin({
title: "Fleet",
contentImage: path.resolve("./assets/images/kolide-logo.svg"),
excludeWarnings: true
})
excludeWarnings: true,
}),
];
if (process.env.NODE_ENV === 'production') {
if (process.env.NODE_ENV === "production") {
plugins = plugins.concat([
new webpack.DefinePlugin({
'process.env': {NODE_ENV: JSON.stringify('production')}
"process.env": { NODE_ENV: JSON.stringify("production") },
}),
new MiniCssExtractPlugin({
filename: "bundle-[contenthash].css",
allChunks: false,
}),
new MiniCssExtractPlugin({ filename: 'bundle-[contenthash].css', allChunks: false })
]);
} else {
// development
plugins = plugins.concat([
new MiniCssExtractPlugin({ filename: 'bundle.css', allChunks: false })
new MiniCssExtractPlugin({ filename: "bundle.css", allChunks: false }),
// Huge speedup on subsequent builds by caching modules
new HardSourceWebpackPlugin({
// Allow pruning anything over an hour old
cachePrune: { maxAge: 60 * 60 * 1000 },
}),
]);
}
var repo = __dirname
var repo = __dirname;
var config = {
var config = {
mode: process.env.NODE_ENV,
entry: {
bundle: path.join(repo, 'frontend/index.jsx')
bundle: path.join(repo, "frontend/index.jsx"),
},
output: {
path: path.join(repo, 'assets/'),
path: path.join(repo, "assets/"),
publicPath: "/assets/",
filename: '[name].js'
filename: "[name].js",
},
devtool: process.env.NODE_ENV === 'development' ? DEV_SOURCE_MAPS : false,
devtool: process.env.NODE_ENV === "development" ? DEV_SOURCE_MAPS : false,
plugins: plugins,
optimization: {
minimize: process.env.NODE_ENV == 'production',
minimize: process.env.NODE_ENV === "production",
},
module: {
// The following noParse suppresses the warning about sqlite-parser being a
// pre-compiled JS file. See https://goo.gl/N4s6bB.
noParse: /node_modules\/sqlite-parser\/dist\/sqlite-parser-min.js/,
rules: [
{ test: /\.(png|gif)$/, use: { loader: 'url-loader?name=[name]@[hash].[ext]&limit=6000' } },
{
test: /\.(png|gif)$/,
use: { loader: "url-loader?name=[name]@[hash].[ext]&limit=6000" },
},
{
test: /\.(pdf|ico|jpg|svg|eot|otf|woff|woff2|ttf|mp4|webm)$/,
use: {
loader: 'file-loader',
loader: "file-loader",
options: {
name: '[name]@[hash].[ext]',
name: "[name]@[hash].[ext]",
useRelativePath: true,
},
},
},
{ test: /\.tsx?$/, exclude: /node_modules/, use: { loader: 'ts-loader' } },
{
test: /\.tsx?$/,
exclude: /node_modules/,
use: { loader: "ts-loader" },
},
{
test: /\.scss$/,
exclude: /node_modules/,
@@ -77,20 +94,20 @@ var config = {
{
loader: MiniCssExtractPlugin.loader,
options: {
publicPath: './',
hmr: process.env.NODE_ENV == 'development',
publicPath: "./",
hmr: process.env.NODE_ENV === "development",
},
},
{ loader: 'css-loader' },
{ loader: 'postcss-loader' },
{ loader: "css-loader" },
{ loader: "postcss-loader" },
{
loader: 'sass-loader',
loader: "sass-loader",
options: {
sourceMap: true,
includePaths: [ bourbon ]
}
includePaths: [bourbon],
},
},
{ loader: 'import-glob-loader' }
{ loader: "import-glob-loader" },
],
},
{
@@ -99,31 +116,28 @@ var config = {
{
loader: MiniCssExtractPlugin.loader,
options: {
hmr: process.env.NODE_ENV == 'development',
hmr: process.env.NODE_ENV === "development",
},
},
'css-loader',
'postcss-loader'
"css-loader",
"postcss-loader",
],
},
{
test: /\.jsx?$/,
include: path.join(repo, 'frontend'),
use: [ 'babel-loader' ]
}
]
include: path.join(repo, "frontend"),
use: { loader: "babel-loader", options: { cacheDirectory: true } },
},
],
},
resolve: {
extensions: ['.tsx', '.ts', '.js', '.jsx', '.json'],
modules: [
path.resolve(path.join(repo, './frontend')),
"node_modules"
]
}
extensions: [".tsx", ".ts", ".js", ".jsx", ".json"],
modules: [path.resolve(path.join(repo, "./frontend")), "node_modules"],
},
};
if (process.env.NODE_ENV === 'production') {
config.output.filename = "[name]-[hash].js"
if (process.env.NODE_ENV === "production") {
config.output.filename = "[name]-[hash].js";
}
module.exports = config;
yarn.lock
+85 -6
View File
@@ -4758,6 +4758,11 @@ detect-indent@^4.0.0:
dependencies:
repeating "^2.0.0"
detect-indent@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-5.0.0.tgz#3871cc0a6a002e8c3e5b3cf7f336264675f06b9d"
integrity sha1-OHHMCmoALow+Wzz38zYmRnXwa50=
detect-libc@^1.0.2:
version "1.0.3"
resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b"
@@ -6481,6 +6486,25 @@ har-validator@~5.1.3:
ajv "^6.12.3"
har-schema "^2.0.0"
hard-source-webpack-plugin@^0.13.1:
version "0.13.1"
resolved "https://registry.yarnpkg.com/hard-source-webpack-plugin/-/hard-source-webpack-plugin-0.13.1.tgz#a99071e25b232f1438a5bc3c99f10a3869e4428e"
integrity sha512-r9zf5Wq7IqJHdVAQsZ4OP+dcUSvoHqDMxJlIzaE2J0TZWn3UjMMrHqwDHR8Jr/pzPfG7XxSe36E7Y8QGNdtuAw==
dependencies:
chalk "^2.4.1"
find-cache-dir "^2.0.0"
graceful-fs "^4.1.11"
lodash "^4.15.0"
mkdirp "^0.5.1"
node-object-hash "^1.2.0"
parse-json "^4.0.0"
pkg-dir "^3.0.0"
rimraf "^2.6.2"
semver "^5.6.0"
tapable "^1.0.0-beta.5"
webpack-sources "^1.0.1"
write-json-file "^2.3.0"
harmony-reflect@^1.4.6:
version "1.6.1"
resolved "https://registry.yarnpkg.com/harmony-reflect/-/harmony-reflect-1.6.1.tgz#c108d4f2bb451efef7a37861fdbdae72c9bdefa9"
@@ -8446,6 +8470,11 @@ lodash@^4.0.0, lodash@^4.17.11, lodash@^4.17.2, lodash@^4.17.3, lodash@^4.17.4,
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.14.tgz#9ce487ae66c96254fe20b599f21b6816028078ba"
integrity sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw==
lodash@^4.15.0, lodash@^4.17.20, lodash@^4.7.0:
version "4.17.21"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
lodash@^4.17.15:
version "4.17.15"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548"
@@ -8456,11 +8485,6 @@ lodash@^4.17.19:
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52"
integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==
lodash@^4.17.20, lodash@^4.7.0:
version "4.17.21"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
log-symbols@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-1.0.2.tgz#376ff7b58ea3086a0f09facc74617eca501e1a18"
@@ -8538,6 +8562,13 @@ lz-string@^1.4.4:
resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26"
integrity sha1-wNjq82BZ9wV5bh40SBHPTEmNOiY=
make-dir@^1.0.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-1.3.0.tgz#79c1033b80515bd6d24ec9933e860ca75ee27f0c"
integrity sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==
dependencies:
pify "^3.0.0"
make-dir@^2.0.0, make-dir@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-2.1.0.tgz#5f0310e18b8be898cc07009295a30ae41e91e6f5"
@@ -9069,6 +9100,11 @@ node-notifier@^8.0.0:
uuid "^8.3.0"
which "^2.0.2"
node-object-hash@^1.2.0:
version "1.4.2"
resolved "https://registry.yarnpkg.com/node-object-hash/-/node-object-hash-1.4.2.tgz#385833d85b229902b75826224f6077be969a9e94"
integrity sha512-UdS4swXs85fCGWWf6t6DMGgpN/vnlKeSGEQ7hJcrs7PBFoxoKLmibc3QRb7fwiYsjdL7PX8iI/TMSlZ90dgHhQ==
node-pre-gyp@^0.12.0:
version "0.12.0"
resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.12.0.tgz#39ba4bb1439da030295f899e3b520b7785766149"
@@ -11206,6 +11242,13 @@ rimraf@2, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.3:
dependencies:
glob "^7.1.3"
rimraf@^2.6.2:
version "2.7.1"
resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec"
integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==
dependencies:
glob "^7.1.3"
rimraf@^3.0.0, rimraf@^3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a"
@@ -11633,6 +11676,13 @@ sort-keys@^1.0.0:
dependencies:
is-plain-obj "^1.0.0"
sort-keys@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/sort-keys/-/sort-keys-2.0.0.tgz#658535584861ec97d730d6cf41822e1f56684128"
integrity sha1-ZYU1WEhh7JfXMNbPQYIuH1ZoQSg=
dependencies:
is-plain-obj "^1.0.0"
source-list-map@^2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/source-list-map/-/source-list-map-2.0.1.tgz#3993bd873bfc48479cca9ea3a547835c7c154b34"
@@ -12142,7 +12192,7 @@ tapable@^0.1.8:
resolved "https://registry.yarnpkg.com/tapable/-/tapable-0.1.10.tgz#29c35707c2b70e50d07482b5d202e8ed446dafd4"
integrity sha1-KcNXB8K3DlDQdIK10gLo7URtr9Q=
tapable@^1.0.0, tapable@^1.1.0:
tapable@^1.0.0, tapable@^1.0.0-beta.5, tapable@^1.1.0:
version "1.1.3"
resolved "https://registry.yarnpkg.com/tapable/-/tapable-1.1.3.tgz#a1fccc06b58db61fd7a45da2da44f5f3a3e67ba2"
integrity sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==
@@ -12963,6 +13013,14 @@ webpack-notifier@1.12.0:
node-notifier "^8.0.0"
strip-ansi "^6.0.0"
webpack-sources@^1.0.1:
version "1.4.3"
resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-1.4.3.tgz#eedd8ec0b928fbf1cbfe994e22d2d890f330a933"
integrity sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==
dependencies:
source-list-map "^2.0.0"
source-map "~0.6.1"
webpack-sources@^1.1.0, webpack-sources@^1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-1.3.0.tgz#2a28dcb9f1f45fe960d8f1493252b5ee6530fa85"
@@ -13162,6 +13220,15 @@ wrappy@1:
resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=
write-file-atomic@^2.0.0:
version "2.4.3"
resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-2.4.3.tgz#1fd2e9ae1df3e75b8d8c367443c692d4ca81f481"
integrity sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==
dependencies:
graceful-fs "^4.1.11"
imurmurhash "^0.1.4"
signal-exit "^3.0.2"
write-file-atomic@^3.0.0:
version "3.0.3"
resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-3.0.3.tgz#56bd5c5a5c70481cd19c571bd39ab965a5de56e8"
@@ -13172,6 +13239,18 @@ write-file-atomic@^3.0.0:
signal-exit "^3.0.2"
typedarray-to-buffer "^3.1.5"
write-json-file@^2.3.0:
version "2.3.0"
resolved "https://registry.yarnpkg.com/write-json-file/-/write-json-file-2.3.0.tgz#2b64c8a33004d54b8698c76d585a77ceb61da32f"
integrity sha1-K2TIozAE1UuGmMdtWFp3zrYdoy8=
dependencies:
detect-indent "^5.0.0"
graceful-fs "^4.1.2"
make-dir "^1.0.0"
pify "^3.0.0"
sort-keys "^2.0.0"
write-file-atomic "^2.0.0"
ws@^7.2.3:
version "7.4.0"
resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.0.tgz#a5dd76a24197940d4a8bb9e0e152bb4503764da7"