From fbc1636994c8a51c0c8bf9576d51a42c949de62f Mon Sep 17 00:00:00 2001 From: Zach Wasserman Date: Thu, 11 Mar 2021 15:55:58 -0800 Subject: [PATCH] Improve E2E testing and test documentation (#448) - Add Makefile commands for E2E testing. - Document E2E testing. - Cleanup other test documentation. --- .github/workflows/test.yml | 10 +- Makefile | 15 ++- cypress.json | 4 +- cypress/integration/login/logging_in.spec.ts | 21 ++- docs/3-Contribution/2-Testing.md | 134 ++++++++++++------- 5 files changed, 126 insertions(+), 58 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0116af9434..9312d86096 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: # Pre-starting dependencies here means they are ready to go when we need them. - name: Start Infra Dependencies # Use & to background this - run: docker-compose up -d mysql redis mailhog & + run: docker-compose up -d mysql_test redis mailhog & - name: Build Fleet run: | @@ -45,12 +45,10 @@ jobs: - name: Run E2E Tests run: | - ./build/fleet prepare db --dev - ./build/fleet serve --dev & + make e2e-reset-db + make e2e-serve & sleep 3 - ./build/fleetctl config set --address https://localhost:8080 - ./build/fleetctl config set --tls-skip-verify true - ./build/fleetctl setup --email=test@fleetdm.com --username=test --password=admin123# --org-name='Fleet Test' + make e2e-setup yarn cypress run --config video=false diff --git a/Makefile b/Makefile index 8d23d506f5..6990457c51 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build clean clean-assets +.PHONY: build clean clean-assets e2e-reset-db e2e-serve e2e-setup export GO111MODULE=on @@ -218,3 +218,16 @@ binary-bundle: xp-fleet xp-fleetctl cd build/binary-bundle && cp windows/fleetctl.exe . && zip fleetctl.exe.zip fleetctl.exe cd build/binary-bundle && shasum -a 256 fleet.zip fleetctl.exe.zip fleetctl-macos.tar.gz fleetctl-windows.tar.gz fleetctl-linux.tar.gz +# Drop, create, and migrate the e2e test database +e2e-reset-db: + docker-compose exec -T mysql_test bash -c 'echo "drop database if exists e2e; create database e2e;" | mysql -uroot -ptoor' + ./build/fleet prepare db --mysql_address=localhost:3307 --mysql_username=root --mysql_password=toor --auth_jwt_key=insecure --mysql_database=e2e + +e2e-setup: + ./build/fleetctl config set --context e2e --address https://localhost:8642 + ./build/fleetctl config set --context e2e --tls-skip-verify true + ./build/fleetctl setup --context e2e --email=test@fleetdm.com --username=test --password=admin123# --org-name='Fleet Test' + +e2e-serve: + ./build/fleet serve --mysql_address=localhost:3307 --mysql_username=root --mysql_password=toor --auth_jwt_key=insecure --mysql_database=e2e --server_address=localhost:8642 + diff --git a/cypress.json b/cypress.json index 0967ef424b..0538de004e 100644 --- a/cypress.json +++ b/cypress.json @@ -1 +1,3 @@ -{} +{ + "baseUrl": "https://localhost:8642" +} diff --git a/cypress/integration/login/logging_in.spec.ts b/cypress/integration/login/logging_in.spec.ts index 94d783d898..f09b98265f 100644 --- a/cypress/integration/login/logging_in.spec.ts +++ b/cypress/integration/login/logging_in.spec.ts @@ -1,6 +1,6 @@ -describe('Searching for a host', () => { - it('Logs into the applications', () => { - cy.visit('https://localhost:8080'); +describe('Login', () => { + it('Logs in successfully', () => { + cy.visit('/'); cy.get(':nth-child(1) > .input-icon-field__input') .type('test@fleetdm.com'); @@ -13,4 +13,19 @@ describe('Searching for a host', () => { cy.url().should('include', '/hosts/manage'); cy.contains('All Hosts'); }); + + it('Fails with invalid password', () => { + cy.visit('/'); + cy.get(':nth-child(1) > .input-icon-field__input') + .type('test@fleetdm.com'); + + cy.get(':nth-child(2) > .input-icon-field__input') + .type('bad_password'); + + cy.get('.button') + .click(); + + cy.url().should('include', '/login'); + cy.contains('username or email and password do not match'); + }); }); diff --git a/docs/3-Contribution/2-Testing.md b/docs/3-Contribution/2-Testing.md index dcbb221a6e..fc5ae55618 100644 --- a/docs/3-Contribution/2-Testing.md +++ b/docs/3-Contribution/2-Testing.md @@ -1,16 +1,14 @@ # Testing -- [Full test suite](#full-test-suite) - - [Database tests](#database-tests) - - [Email tests](#email-tests) -- [Integration tests](#integration-tests) - - [Email](#email) +- [Test suite](#full-test-suite) +- [End-to-end tests](#end-to-end-tests) +- [Email](#email) -## Full test suite +## Test suite -To execute all of the tests that CI will execute, run the following from the root of the repository: +To execute the basic unit tests, run the following from the root of the repository: ``` -make test +MYSQL_TEST=1 REDIS_TEST=1 make test ``` It is a good idea to run `make test` before submitting a Pull Request. @@ -23,14 +21,6 @@ To run all Go unit tests, run the following: make test-go ``` -#### JavaScript unit tests - -To run all JavaScript unit tests, run the following: - -``` -make test-js -``` - #### Go linters To run all Go linters and static analyzers, run the following: @@ -39,44 +29,59 @@ To run all Go linters and static analyzers, run the following: make lint-go ``` -### Database tests +#### Javascript unit tests -To run database tests set environment variables as follows. +To run all JS unit tests, run the following: ``` -export MYSQL_PORT_3306_TCP_ADDR=192.168.99.100 -export MYSQL_TEST=1 +make test-js ``` -### Email tests - -To run email related unit tests using MailHog set the following environment -variable. +or ``` -export MAIL_TEST=1 +yarn test ``` -## Integration tests +#### Javascript linters -By default, tests that require external dependecies like Mysql or Redis are skipped. The tests can be enabled by setting `MYSQL_TEST=true` and `REDIS_TEST=true` environment variables. MYSQL will try to connect with the following credentials. -``` -user = "kolide" -password = "kolide" -database = "kolide" -host = "127.0.0.1" -``` -Redis tests expect a redis instance at `127.0.0.1:6379`. - -#### JavaScript linters - -To run all JavaScript linters and static analyzers, run the following: +To run all JS linters and static analyzers, run the following: ``` make lint-js ``` -#### Viewing test coverage +or + +``` +yarn lint +``` + +#### MySQL tests + +To run MySQL integration tests set environment variables as follows: + +``` +MYSQL_TEST=1 make test-go +``` + +#### Redis tests + +To run Redis integration tests set environment variables as follows: + +``` +REDIS_TEST=1 make test-go +``` + +#### Email tests + +To run email related integration tests using MailHog set environment as follows: + +``` +MAIL_TEST=1 make test-go +``` + +### Viewing test coverage When you run `make test` or `make test-go` from the root of the repository, test coverage reports are generated in every subpackage. For example, the `server` subpackage will have a coverage report generated in `./server/server.cover` @@ -94,17 +99,52 @@ To view test a test coverage report in a terminal, run the following: go tool cover -func=./server/server.cover ``` -### Email +## End-to-end tests -#### Testing email using MailHog +E2E tests are run using Docker and Cypress. -To intercept sent emails while running a Fleet development environment, make sure that you've set the SMTP address to `:1025` and leave the username and password blank. Then, visit `:8025` in a web browser to view the [MailHog](https://github.com/mailhog/MailHog) UI. +#### Preparation -For example, if docker is running natively on your `localhost`, then your mail settings should look something like: +Make sure dependencies are up to date and the [Fleet binaries are built locally](./1-Building-Fleet.md). -```yaml -mail: - address: localhost:1025 +``` +make e2e-reset-db +make e2e-serve ``` -`localhost:1025` is the default configuration. You can use `fleet config_dump` to see the values which Fleet is using given your configuration. +This will start a local Fleet server connected to the E2E database. Leave this server running for the duration of end-to-end testing. + +``` +make e2e-setup +``` + +This will initialize the E2E instance with a user. + +#### Run tests + +Tests can be run in interactive mode, or from the command line. + +#### Interactive + +``` +yarn cypress open +``` + +Use the graphical UI controls to run and view tests. + +#### Command line + +``` +yarn cypress run +``` + +Tests will run automatically and results are reported to the shell. + + +## Email + +#### Manually testing email with MailHog + +To intercept sent emails while running a Fleet development environment, make sure that you've set the SMTP address to `localhost:1025` and leave the username and password blank. Then, visit http://localhost:8025 in a web browser to view the [MailHog](https://github.com/mailhog/MailHog) UI. + +When Fleet sends emails, the contents of the messages are available in the MailHog UI.