diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8b784ac9cd..67075467dd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,21 +15,42 @@ jobs: uses: actions/checkout@v2 - name: JS Dependency Cache + id: js-cache uses: actions/cache@v2 with: path: | **/node_modules ~/.cache/Cypress - key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }} + # Use a separate cache for this from other JS jobs since we run the + # webpack steps and will have more to cache. + key: ${{ runner.os }}-e2e-modules-${{ hashFiles('**/yarn.lock') }} restore-keys: | - ${{ runner.os }}-modules- + ${{ runner.os }}-e2e-modules- - # It seems faster not to cache Go dependencies + - name: Go Cache + id: go-cache + uses: actions/cache@v2 + with: + # In order: + # * Module download cache + # * Build cache (Linux) + # * Build cache (Mac) + # * Build cache (Windows) + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- - name: Install JS Dependencies + if: steps.js-cache.outputs.cache-hit != 'true' run: make deps-js - name: Install Go Dependencies + if: steps.go-cache.outputs.cache-hit != 'true' run: make deps-go # Pre-starting dependencies here means they are ready to go when we need them. @@ -40,7 +61,7 @@ jobs: - name: Build Fleet run: | export PATH=$PATH:~/go/bin - make generate + make generate-ci make - name: Run E2E Tests @@ -63,6 +84,7 @@ jobs: uses: actions/checkout@v2 - name: JS Dependency Cache + id: js-cache uses: actions/cache@v2 with: path: | @@ -73,6 +95,7 @@ jobs: ${{ runner.os }}-modules- - name: Install JS Dependencies + if: steps.js-cache.outputs.cache-hit != 'true' run: make deps-js - name: Run JS Tests @@ -91,6 +114,7 @@ jobs: uses: actions/checkout@v2 - name: JS Dependency Cache + id: js-cache uses: actions/cache@v2 with: path: | @@ -101,38 +125,16 @@ jobs: ${{ runner.os }}-modules- - name: Install JS Dependencies + if: steps.js-cache.outputs.cache-hit != 'true' run: make deps-js - name: Run JS Linting run: | make lint-js - check-prettier: - strategy: - matrix: - os: [ ubuntu-latest ] - runs-on: ${{ matrix.os }} - - steps: - - name: Checkout Code - uses: actions/checkout@v2 - - - name: JS Dependency Cache - uses: actions/cache@v2 - with: - path: | - **/node_modules - ~/.cache/Cypress - key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-modules- - - - name: Install JS Dependencies - run: make deps-js - - - name: Run prettier formatting check - run: | - yarn prettier:check + - name: Run prettier formatting check + run: | + yarn prettier:check test-go: strategy: diff --git a/Makefile b/Makefile index 88b9abeb3f..5f48df73ec 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ ifneq ($(OS), Windows_NT) # To populate version metadata, we use unix tools to get certain data GOVERSION = $(shell go version | awk '{print $$3}') - NOW = $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") + NOW = $(shell date +"%Y-%m-%d") else # The output binary name is different on Windows, so we're explicit here OUTPUT = fleet.exe @@ -29,7 +29,7 @@ else # To populate version metadata, we use windows tools to get the certain data GOVERSION_CMD = "(go version).Split()[2]" GOVERSION = $(shell powershell $(GOVERSION_CMD)) - NOW = $(shell powershell Get-Date -format s) + NOW = $(shell powershell Get-Date -format "yyy-MM-dd") endif ifndef CIRCLE_PR_NUMBER @@ -131,6 +131,10 @@ test: lint test-go test-js generate: clean-assets generate-js generate-go +generate-ci: + NODE_ENV=development webpack + make generate-go + generate-js: clean-assets .prefix NODE_ENV=production webpack --progress --colors diff --git a/package.json b/package.json index 7db602028c..17dad6ca43 100644 --- a/package.json +++ b/package.json @@ -174,6 +174,7 @@ "eslint-plugin-react-hooks": "^4.2.0", "expose-loader": "0.7.5", "file-loader": "^4.1.0", + "hard-source-webpack-plugin": "^0.13.1", "html-webpack-plugin": "^3.2.0", "identity-obj-proxy": "^3.0.0", "ignore-styles": "5.0.1", diff --git a/webpack.config.js b/webpack.config.js index 6c5dd2b50c..28a80ce759 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -1,75 +1,92 @@ -require('es6-promise').polyfill(); +require("es6-promise").polyfill(); -const path = require('path'); -const webpack = require('webpack'); -const bourbon = require('node-bourbon').includePaths; -const HtmlWebpackPlugin = require('html-webpack-plugin'); -const MiniCssExtractPlugin = require('mini-css-extract-plugin'); -const WebpackNotifierPlugin = require('webpack-notifier'); +const path = require("path"); +const webpack = require("webpack"); +const bourbon = require("node-bourbon").includePaths; +const HtmlWebpackPlugin = require("html-webpack-plugin"); +const MiniCssExtractPlugin = require("mini-css-extract-plugin"); +const WebpackNotifierPlugin = require("webpack-notifier"); +const HardSourceWebpackPlugin = require("hard-source-webpack-plugin"); -const DEV_SOURCE_MAPS = 'eval-source-map'; +const DEV_SOURCE_MAPS = "eval-source-map"; var plugins = [ new webpack.NoEmitOnErrorsPlugin(), new HtmlWebpackPlugin({ - filename: '../frontend/templates/react.tmpl', + filename: "../frontend/templates/react.tmpl", inject: false, - template: 'frontend/templates/react.ejs' + template: "frontend/templates/react.ejs", }), new WebpackNotifierPlugin({ title: "Fleet", contentImage: path.resolve("./assets/images/kolide-logo.svg"), - excludeWarnings: true - }) + excludeWarnings: true, + }), ]; -if (process.env.NODE_ENV === 'production') { +if (process.env.NODE_ENV === "production") { plugins = plugins.concat([ new webpack.DefinePlugin({ - 'process.env': {NODE_ENV: JSON.stringify('production')} + "process.env": { NODE_ENV: JSON.stringify("production") }, + }), + new MiniCssExtractPlugin({ + filename: "bundle-[contenthash].css", + allChunks: false, }), - new MiniCssExtractPlugin({ filename: 'bundle-[contenthash].css', allChunks: false }) ]); } else { + // development plugins = plugins.concat([ - new MiniCssExtractPlugin({ filename: 'bundle.css', allChunks: false }) + new MiniCssExtractPlugin({ filename: "bundle.css", allChunks: false }), + // Huge speedup on subsequent builds by caching modules + new HardSourceWebpackPlugin({ + // Allow pruning anything over an hour old + cachePrune: { maxAge: 60 * 60 * 1000 }, + }), ]); } -var repo = __dirname +var repo = __dirname; -var config = { +var config = { mode: process.env.NODE_ENV, entry: { - bundle: path.join(repo, 'frontend/index.jsx') + bundle: path.join(repo, "frontend/index.jsx"), }, output: { - path: path.join(repo, 'assets/'), + path: path.join(repo, "assets/"), publicPath: "/assets/", - filename: '[name].js' + filename: "[name].js", }, - devtool: process.env.NODE_ENV === 'development' ? DEV_SOURCE_MAPS : false, + devtool: process.env.NODE_ENV === "development" ? DEV_SOURCE_MAPS : false, plugins: plugins, optimization: { - minimize: process.env.NODE_ENV == 'production', + minimize: process.env.NODE_ENV === "production", }, module: { // The following noParse suppresses the warning about sqlite-parser being a // pre-compiled JS file. See https://goo.gl/N4s6bB. noParse: /node_modules\/sqlite-parser\/dist\/sqlite-parser-min.js/, rules: [ - { test: /\.(png|gif)$/, use: { loader: 'url-loader?name=[name]@[hash].[ext]&limit=6000' } }, + { + test: /\.(png|gif)$/, + use: { loader: "url-loader?name=[name]@[hash].[ext]&limit=6000" }, + }, { test: /\.(pdf|ico|jpg|svg|eot|otf|woff|woff2|ttf|mp4|webm)$/, use: { - loader: 'file-loader', + loader: "file-loader", options: { - name: '[name]@[hash].[ext]', + name: "[name]@[hash].[ext]", useRelativePath: true, }, }, }, - { test: /\.tsx?$/, exclude: /node_modules/, use: { loader: 'ts-loader' } }, + { + test: /\.tsx?$/, + exclude: /node_modules/, + use: { loader: "ts-loader" }, + }, { test: /\.scss$/, exclude: /node_modules/, @@ -77,20 +94,20 @@ var config = { { loader: MiniCssExtractPlugin.loader, options: { - publicPath: './', - hmr: process.env.NODE_ENV == 'development', + publicPath: "./", + hmr: process.env.NODE_ENV === "development", }, }, - { loader: 'css-loader' }, - { loader: 'postcss-loader' }, + { loader: "css-loader" }, + { loader: "postcss-loader" }, { - loader: 'sass-loader', + loader: "sass-loader", options: { sourceMap: true, - includePaths: [ bourbon ] - } + includePaths: [bourbon], + }, }, - { loader: 'import-glob-loader' } + { loader: "import-glob-loader" }, ], }, { @@ -99,31 +116,28 @@ var config = { { loader: MiniCssExtractPlugin.loader, options: { - hmr: process.env.NODE_ENV == 'development', + hmr: process.env.NODE_ENV === "development", }, }, - 'css-loader', - 'postcss-loader' + "css-loader", + "postcss-loader", ], }, { test: /\.jsx?$/, - include: path.join(repo, 'frontend'), - use: [ 'babel-loader' ] - } - ] + include: path.join(repo, "frontend"), + use: { loader: "babel-loader", options: { cacheDirectory: true } }, + }, + ], }, resolve: { - extensions: ['.tsx', '.ts', '.js', '.jsx', '.json'], - modules: [ - path.resolve(path.join(repo, './frontend')), - "node_modules" - ] - } + extensions: [".tsx", ".ts", ".js", ".jsx", ".json"], + modules: [path.resolve(path.join(repo, "./frontend")), "node_modules"], + }, }; -if (process.env.NODE_ENV === 'production') { - config.output.filename = "[name]-[hash].js" +if (process.env.NODE_ENV === "production") { + config.output.filename = "[name]-[hash].js"; } module.exports = config; diff --git a/yarn.lock b/yarn.lock index dd2da51460..38012508fd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4758,6 +4758,11 @@ detect-indent@^4.0.0: dependencies: repeating "^2.0.0" +detect-indent@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-5.0.0.tgz#3871cc0a6a002e8c3e5b3cf7f336264675f06b9d" + integrity sha1-OHHMCmoALow+Wzz38zYmRnXwa50= + detect-libc@^1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b" @@ -6481,6 +6486,25 @@ har-validator@~5.1.3: ajv "^6.12.3" har-schema "^2.0.0" +hard-source-webpack-plugin@^0.13.1: + version "0.13.1" + resolved "https://registry.yarnpkg.com/hard-source-webpack-plugin/-/hard-source-webpack-plugin-0.13.1.tgz#a99071e25b232f1438a5bc3c99f10a3869e4428e" + integrity sha512-r9zf5Wq7IqJHdVAQsZ4OP+dcUSvoHqDMxJlIzaE2J0TZWn3UjMMrHqwDHR8Jr/pzPfG7XxSe36E7Y8QGNdtuAw== + dependencies: + chalk "^2.4.1" + find-cache-dir "^2.0.0" + graceful-fs "^4.1.11" + lodash "^4.15.0" + mkdirp "^0.5.1" + node-object-hash "^1.2.0" + parse-json "^4.0.0" + pkg-dir "^3.0.0" + rimraf "^2.6.2" + semver "^5.6.0" + tapable "^1.0.0-beta.5" + webpack-sources "^1.0.1" + write-json-file "^2.3.0" + harmony-reflect@^1.4.6: version "1.6.1" resolved "https://registry.yarnpkg.com/harmony-reflect/-/harmony-reflect-1.6.1.tgz#c108d4f2bb451efef7a37861fdbdae72c9bdefa9" @@ -8446,6 +8470,11 @@ lodash@^4.0.0, lodash@^4.17.11, lodash@^4.17.2, lodash@^4.17.3, lodash@^4.17.4, resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.14.tgz#9ce487ae66c96254fe20b599f21b6816028078ba" integrity sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw== +lodash@^4.15.0, lodash@^4.17.20, lodash@^4.7.0: + version "4.17.21" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" + integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== + lodash@^4.17.15: version "4.17.15" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548" @@ -8456,11 +8485,6 @@ lodash@^4.17.19: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA== -lodash@^4.17.20, lodash@^4.7.0: - version "4.17.21" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" - integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== - log-symbols@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-1.0.2.tgz#376ff7b58ea3086a0f09facc74617eca501e1a18" @@ -8538,6 +8562,13 @@ lz-string@^1.4.4: resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26" integrity sha1-wNjq82BZ9wV5bh40SBHPTEmNOiY= +make-dir@^1.0.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-1.3.0.tgz#79c1033b80515bd6d24ec9933e860ca75ee27f0c" + integrity sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ== + dependencies: + pify "^3.0.0" + make-dir@^2.0.0, make-dir@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-2.1.0.tgz#5f0310e18b8be898cc07009295a30ae41e91e6f5" @@ -9069,6 +9100,11 @@ node-notifier@^8.0.0: uuid "^8.3.0" which "^2.0.2" +node-object-hash@^1.2.0: + version "1.4.2" + resolved "https://registry.yarnpkg.com/node-object-hash/-/node-object-hash-1.4.2.tgz#385833d85b229902b75826224f6077be969a9e94" + integrity sha512-UdS4swXs85fCGWWf6t6DMGgpN/vnlKeSGEQ7hJcrs7PBFoxoKLmibc3QRb7fwiYsjdL7PX8iI/TMSlZ90dgHhQ== + node-pre-gyp@^0.12.0: version "0.12.0" resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.12.0.tgz#39ba4bb1439da030295f899e3b520b7785766149" @@ -11206,6 +11242,13 @@ rimraf@2, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.3: dependencies: glob "^7.1.3" +rimraf@^2.6.2: + version "2.7.1" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" + integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w== + dependencies: + glob "^7.1.3" + rimraf@^3.0.0, rimraf@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" @@ -11633,6 +11676,13 @@ sort-keys@^1.0.0: dependencies: is-plain-obj "^1.0.0" +sort-keys@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/sort-keys/-/sort-keys-2.0.0.tgz#658535584861ec97d730d6cf41822e1f56684128" + integrity sha1-ZYU1WEhh7JfXMNbPQYIuH1ZoQSg= + dependencies: + is-plain-obj "^1.0.0" + source-list-map@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/source-list-map/-/source-list-map-2.0.1.tgz#3993bd873bfc48479cca9ea3a547835c7c154b34" @@ -12142,7 +12192,7 @@ tapable@^0.1.8: resolved "https://registry.yarnpkg.com/tapable/-/tapable-0.1.10.tgz#29c35707c2b70e50d07482b5d202e8ed446dafd4" integrity sha1-KcNXB8K3DlDQdIK10gLo7URtr9Q= -tapable@^1.0.0, tapable@^1.1.0: +tapable@^1.0.0, tapable@^1.0.0-beta.5, tapable@^1.1.0: version "1.1.3" resolved "https://registry.yarnpkg.com/tapable/-/tapable-1.1.3.tgz#a1fccc06b58db61fd7a45da2da44f5f3a3e67ba2" integrity sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA== @@ -12963,6 +13013,14 @@ webpack-notifier@1.12.0: node-notifier "^8.0.0" strip-ansi "^6.0.0" +webpack-sources@^1.0.1: + version "1.4.3" + resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-1.4.3.tgz#eedd8ec0b928fbf1cbfe994e22d2d890f330a933" + integrity sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ== + dependencies: + source-list-map "^2.0.0" + source-map "~0.6.1" + webpack-sources@^1.1.0, webpack-sources@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-1.3.0.tgz#2a28dcb9f1f45fe960d8f1493252b5ee6530fa85" @@ -13162,6 +13220,15 @@ wrappy@1: resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= +write-file-atomic@^2.0.0: + version "2.4.3" + resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-2.4.3.tgz#1fd2e9ae1df3e75b8d8c367443c692d4ca81f481" + integrity sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ== + dependencies: + graceful-fs "^4.1.11" + imurmurhash "^0.1.4" + signal-exit "^3.0.2" + write-file-atomic@^3.0.0: version "3.0.3" resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-3.0.3.tgz#56bd5c5a5c70481cd19c571bd39ab965a5de56e8" @@ -13172,6 +13239,18 @@ write-file-atomic@^3.0.0: signal-exit "^3.0.2" typedarray-to-buffer "^3.1.5" +write-json-file@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/write-json-file/-/write-json-file-2.3.0.tgz#2b64c8a33004d54b8698c76d585a77ceb61da32f" + integrity sha1-K2TIozAE1UuGmMdtWFp3zrYdoy8= + dependencies: + detect-indent "^5.0.0" + graceful-fs "^4.1.2" + make-dir "^1.0.0" + pify "^3.0.0" + sort-keys "^2.0.0" + write-file-atomic "^2.0.0" + ws@^7.2.3: version "7.4.0" resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.0.tgz#a5dd76a24197940d4a8bb9e0e152bb4503764da7"