From d70a0c8c24944370403521a4f3f3f30b1f0b106b Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Wed, 25 Oct 2017 10:22:18 -0400 Subject: [PATCH 1/2] cleanup e2e test development tooling and add readme examples --- awx/ui/test/e2e/README.md | 25 +++++++++++++------ awx/ui/test/e2e/cluster/Dockerfile | 21 ++++++++++++++++ awx/ui/test/e2e/cluster/devel-override.yml | 13 ---------- .../cluster/docker-compose.devel-override.yml | 18 +++++++++++++ awx/ui/test/e2e/cluster/docker-compose.yml | 16 +++++++++++- 5 files changed, 71 insertions(+), 22 deletions(-) create mode 100644 awx/ui/test/e2e/cluster/Dockerfile delete mode 100644 awx/ui/test/e2e/cluster/devel-override.yml create mode 100644 awx/ui/test/e2e/cluster/docker-compose.devel-override.yml diff --git a/awx/ui/test/e2e/README.md b/awx/ui/test/e2e/README.md index b4c1a0aab8..86ff61f36b 100644 --- a/awx/ui/test/e2e/README.md +++ b/awx/ui/test/e2e/README.md @@ -7,17 +7,26 @@ docker exec -i tools_awx_1 sh <<-EOSH make --directory=/awx_devel DATA_GEN_PRESET=e2e bulk_data EOSH -# run with with a live browser -npm --prefix awx/ui run e2e -- --env=debug +# run all of the tests with a live browser +npm --prefix awx/ui run e2e + +# run a subset of the tests +npm --prefix awx/ui run e2e -- --filter="test-credentials*" # setup a local webdriver cluster for test development docker-compose \ - -f awx/ui/client/test/e2e/cluster/docker-compose.yml \ - -f awx/ui/client/test/e2e/cluster/devel-override.yml \ - up --scale chrome=2 --scale firefox=0 + -f awx/ui/test/e2e/cluster/docker-compose.yml \ + -f awx/ui/test/e2e/cluster/docker-compose.devel-override.yml \ + up --scale chrome=2 hub chrome -# run headlessly with multiple workers on the cluster -AWX_E2E_LAUNCH_URL='https://awx:8043' AWX_E2E_WORKERS=2 npm --prefix awx/ui run e2e +# run headlessly on the cluster +AWX_E2E_LAUNCH_URL='https://awx:8043' npm --prefix awx/ui run e2e -- --env=cluster + +# run with multiple workers +AWX_E2E_LAUNCH_URL='https://awx:8043' AWX_E2E_CLUSTER_WORKERS=2 \ + npm --prefix awx/ui run e2e -- --env=cluster --filter="test-*" ``` -**Note:** Unless overridden in [settings](settings.js), tests will run against `localhost:8043`. +**Note:** +- Unless overridden in [settings](settings.js), tests will run against `localhost:8043`. +- Use `npm --prefix awx/ui run e2e -- --help` to see additional usage information for the test runner. diff --git a/awx/ui/test/e2e/cluster/Dockerfile b/awx/ui/test/e2e/cluster/Dockerfile new file mode 100644 index 0000000000..0c92d037fe --- /dev/null +++ b/awx/ui/test/e2e/cluster/Dockerfile @@ -0,0 +1,21 @@ +FROM centos:7 + +RUN yum install -y epel-release + +RUN yum install -y \ + bzip2 \ + gcc-c++ \ + git \ + make \ + nodejs \ + npm + +WORKDIR /awx + +COPY awx/ui/package.json awx/ui/package.json + +RUN npm --prefix=awx/ui install + +COPY awx/ui/test/e2e awx/ui/test/e2e + +ENTRYPOINT ["npm", "--prefix=awx/ui", "run", "e2e", "--", "--env=cluster"] diff --git a/awx/ui/test/e2e/cluster/devel-override.yml b/awx/ui/test/e2e/cluster/devel-override.yml deleted file mode 100644 index dc399d5869..0000000000 --- a/awx/ui/test/e2e/cluster/devel-override.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -version: '2' -networks: - default: - external: - name: tools_default -services: - chrome: - external_links: - - 'tools_awx_1:awx' - firefox: - external_links: - - 'tools_awx_1:awx' diff --git a/awx/ui/test/e2e/cluster/docker-compose.devel-override.yml b/awx/ui/test/e2e/cluster/docker-compose.devel-override.yml new file mode 100644 index 0000000000..eed4e15a94 --- /dev/null +++ b/awx/ui/test/e2e/cluster/docker-compose.devel-override.yml @@ -0,0 +1,18 @@ +--- +version: '2' +networks: + default: + external: + name: tools_default +services: + chrome: + external_links: + - tools_awx_1:awx + firefox: + external_links: + - tools_awx_1:awx + e2e: + external_links: + - tools_awx_1:awx + environment: + AWX_E2E_URL: https://awx:8043 diff --git a/awx/ui/test/e2e/cluster/docker-compose.yml b/awx/ui/test/e2e/cluster/docker-compose.yml index 8ac55ce20d..f5c5336d37 100644 --- a/awx/ui/test/e2e/cluster/docker-compose.yml +++ b/awx/ui/test/e2e/cluster/docker-compose.yml @@ -4,7 +4,7 @@ services: hub: image: selenium/hub ports: - - '4444:4444' + - 4444:4444 chrome: image: selenium/node-chrome links: @@ -21,3 +21,17 @@ services: environment: HUB_PORT_4444_TCP_ADDR: hub HUB_PORT_4444_TCP_PORT: 4444 + e2e: + image: awx_e2e + build: + context: ../../../../../ + dockerfile: awx/ui/test/e2e/cluster/Dockerfile + depends_on: + - chrome + links: + - hub + volumes: + - ..:/awx/awx/ui/test/e2e + environment: + AWX_E2E_CLUSTER_HOST: hub + AWX_E2E_CLUSTER_PORT: 4444 From 3800a16f3e53397eb80571b64756395071863923 Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Wed, 25 Oct 2017 10:27:39 -0400 Subject: [PATCH 2/2] refactor e2e settings and config modules This should make the settings and configuration logic less implicit and a little easier to follow. Some familiarity with the configuration behavior of nightwatch is still necessary in places - specifically, one should know that all test_settings defined for non-default environments are treated as overrides to the values defined for the default environment. --- awx/ui/package.json | 2 +- awx/ui/test/e2e/api.js | 14 +++--- awx/ui/test/e2e/commands/login.js | 12 +++-- awx/ui/test/e2e/commands/waitForAngular.js | 4 +- awx/ui/test/e2e/nightwatch.conf.js | 49 +++++++++++-------- awx/ui/test/e2e/settings.js | 43 +++++++--------- .../tests/test-credentials-add-edit-gce.js | 4 +- .../test-credentials-add-edit-insights.js | 4 +- .../test-credentials-add-edit-machine.js | 4 +- .../test-credentials-add-edit-network.js | 4 +- .../tests/test-credentials-add-edit-scm.js | 4 +- .../tests/test-credentials-add-edit-vault.js | 5 +- 12 files changed, 87 insertions(+), 62 deletions(-) diff --git a/awx/ui/package.json b/awx/ui/package.json index 971f206e15..fc04cf9f75 100644 --- a/awx/ui/package.json +++ b/awx/ui/package.json @@ -24,7 +24,7 @@ "test": "karma start test/spec/karma.spec.js", "jshint": "grunt jshint:source --no-color", "test:ci": "npm run test -- --single-run --reporter junit,dots --browsers=PhantomJS", - "e2e": "./test/e2e/runner.js --config ./test/e2e/nightwatch.conf.js", + "e2e": "./test/e2e/runner.js --config ./test/e2e/nightwatch.conf.js --suiteRetries=2", "unit": "karma start test/unit/karma.unit.js", "lint": "eslint .", "dev": "webpack --config build/webpack.development.js --progress", diff --git a/awx/ui/test/e2e/api.js b/awx/ui/test/e2e/api.js index 90b29c4355..ba03b1fd21 100644 --- a/awx/ui/test/e2e/api.js +++ b/awx/ui/test/e2e/api.js @@ -3,15 +3,15 @@ import https from 'https'; import axios from 'axios'; import { - awxURL, - awxUsername, - awxPassword + AWX_E2E_URL, + AWX_E2E_USERNAME, + AWX_E2E_PASSWORD } from './settings'; let authenticated; const session = axios.create({ - baseURL: awxURL, + baseURL: AWX_E2E_URL, xsrfHeaderName: 'X-CSRFToken', xsrfCookieName: 'csrftoken', httpsAgent: new https.Agent({ @@ -24,7 +24,7 @@ const getEndpoint = location => { return location; } - return `${awxURL}/api/v2${location}`; + return `${AWX_E2E_URL}/api/v2${location}`; }; const authenticate = () => { @@ -35,8 +35,8 @@ const authenticate = () => { const uri = getEndpoint('/authtoken/'); const credentials = { - username: awxUsername, - password: awxPassword + username: AWX_E2E_USERNAME, + password: AWX_E2E_PASSWORD }; return session.post(uri, credentials).then(res => { diff --git a/awx/ui/test/e2e/commands/login.js b/awx/ui/test/e2e/commands/login.js index 49acc34553..65c9cce1ea 100644 --- a/awx/ui/test/e2e/commands/login.js +++ b/awx/ui/test/e2e/commands/login.js @@ -1,6 +1,12 @@ import { EventEmitter } from 'events'; import { inherits } from 'util'; +import { + AWX_E2E_USERNAME, + AWX_E2E_PASSWORD, + AWX_E2E_TIMEOUT_LONG +} from '../settings'; + function Login () { EventEmitter.call(this); } @@ -8,14 +14,14 @@ function Login () { inherits(Login, EventEmitter); Login.prototype.command = function command (username, password) { - username = username || this.api.globals.awxUsername; - password = password || this.api.globals.awxPassword; + username = username || AWX_E2E_USERNAME; + password = password || AWX_E2E_PASSWORD; const loginPage = this.api.page.login(); loginPage .navigate() - .waitForElementVisible('@submit', this.api.globals.longWait) + .waitForElementVisible('@submit', AWX_E2E_TIMEOUT_LONG) .waitForElementNotVisible('div.spinny') .setValue('@username', username) .setValue('@password', password) diff --git a/awx/ui/test/e2e/commands/waitForAngular.js b/awx/ui/test/e2e/commands/waitForAngular.js index 3f1acf5cd6..bad16b09cf 100644 --- a/awx/ui/test/e2e/commands/waitForAngular.js +++ b/awx/ui/test/e2e/commands/waitForAngular.js @@ -1,5 +1,7 @@ +import { AWX_E2E_TIMEOUT_ASYNC } from '../settings'; + exports.command = function waitForAngular (callback) { - this.timeoutsAsyncScript(this.globals.asyncHookTimeout, () => { + this.timeoutsAsyncScript(AWX_E2E_TIMEOUT_ASYNC, () => { this.executeAsync(done => { if (angular && angular.getTestability) { angular.getTestability(document.body).whenStable(done); diff --git a/awx/ui/test/e2e/nightwatch.conf.js b/awx/ui/test/e2e/nightwatch.conf.js index ab5bb24ef0..67566c8932 100644 --- a/awx/ui/test/e2e/nightwatch.conf.js +++ b/awx/ui/test/e2e/nightwatch.conf.js @@ -2,46 +2,55 @@ import path from 'path'; import chromedriver from 'chromedriver'; -import { test_workers } from './settings.js'; - +import { + AWX_E2E_CLUSTER_HOST, + AWX_E2E_CLUSTER_PORT, + AWX_E2E_CLUSTER_WORKERS, + AWX_E2E_LAUNCH_URL, + AWX_E2E_TIMEOUT_ASYNC, + AWX_E2E_TIMEOUT_MEDIUM +} from './settings'; const resolve = location => path.resolve(__dirname, location); - module.exports = { src_folders: [resolve('tests')], output_folder: resolve('reports'), custom_commands_path: resolve('commands'), page_objects_path: resolve('objects'), - globals_path: resolve('settings.js'), test_settings: { default: { - test_workers, - skip_testcases_on_fail: false, - desiredCapabilities: { - browserName: 'chrome' - } - }, - smoke: { - disable_colors: true, - skip_testcases_on_fail: true, - filter: 'smoke.js' - }, - debug: { - selenium_port: 9515, selenium_host: 'localhost', + selenium_port: 9515, default_path_prefix: '', + desiredCapabilities: { browserName: 'chrome' }, test_workers: { enabled: false }, globals: { - before(done) { - chromedriver.start(); + launch_url: AWX_E2E_LAUNCH_URL, + retryAssertionTimeout: AWX_E2E_TIMEOUT_MEDIUM, + waitForConditionTimeout: AWX_E2E_TIMEOUT_MEDIUM, + asyncHookTimeout: AWX_E2E_TIMEOUT_ASYNC, + before (done) { + chromedriver.start(['--port=9515']); done(); }, - after(done) { + after (done) { chromedriver.stop(); done(); } } + }, + // Note: These are environment-specific overrides to the default + // test settings defined above. + cluster: { + selenium_host: AWX_E2E_CLUSTER_HOST, + selenium_port: AWX_E2E_CLUSTER_PORT, + default_path_prefix: '/wd/hub', + test_workers: { + enabled: (AWX_E2E_CLUSTER_WORKERS > 0), + workers: AWX_E2E_CLUSTER_WORKERS + }, + globals: { before: {}, after: {} } } } }; diff --git a/awx/ui/test/e2e/settings.js b/awx/ui/test/e2e/settings.js index f3bbf1f4b2..a68f9fcc22 100644 --- a/awx/ui/test/e2e/settings.js +++ b/awx/ui/test/e2e/settings.js @@ -1,30 +1,25 @@ +const AWX_E2E_CLUSTER_HOST = process.env.AWX_E2E_CLUSTER_HOST || 'localhost'; +const AWX_E2E_CLUSTER_PORT = process.env.AWX_E2E_CLUSTER_PORT || 4444; +const AWX_E2E_CLUSTER_WORKERS = process.env.AWX_E2E_CLUSTER_WORKERS || 0; +const AWX_E2E_PASSWORD = process.env.AWX_E2E_PASSWORD || 'password'; const AWX_E2E_URL = process.env.AWX_E2E_URL || 'https://localhost:8043'; const AWX_E2E_USERNAME = process.env.AWX_E2E_USERNAME || 'awx-e2e'; -const AWX_E2E_PASSWORD = process.env.AWX_E2E_PASSWORD || 'password'; -const AWX_E2E_SELENIUM_HOST = process.env.AWX_E2E_SELENIUM_HOST || 'localhost'; -const AWX_E2E_SELENIUM_PORT = process.env.AWX_E2E_SELENIUM_PORT || 4444; -const AWX_E2E_LAUNCH_URL = process.env.AWX_E2E_LAUNCH_URL || AWX_E2E_URL; -const AWX_E2E_TIMEOUT_SHORT = process.env.AWX_E2E_TIMEOUT_SHORT || 1000; -const AWX_E2E_TIMEOUT_MEDIUM = process.env.AWX_E2E_TIMEOUT_MEDIUM || 5000; -const AWX_E2E_TIMEOUT_LONG = process.env.AWX_E2E_TIMEOUT_LONG || 10000; const AWX_E2E_TIMEOUT_ASYNC = process.env.AWX_E2E_TIMEOUT_ASYNC || 30000; -const AWX_E2E_WORKERS = process.env.AWX_E2E_WORKERS || 0; +const AWX_E2E_TIMEOUT_LONG = process.env.AWX_E2E_TIMEOUT_LONG || 10000; +const AWX_E2E_TIMEOUT_MEDIUM = process.env.AWX_E2E_TIMEOUT_MEDIUM || 5000; +const AWX_E2E_TIMEOUT_SHORT = process.env.AWX_E2E_TIMEOUT_SHORT || 1000; +const AWX_E2E_LAUNCH_URL = process.env.AWX_E2E_LAUNCH_URL || AWX_E2E_URL; module.exports = { - awxURL: AWX_E2E_URL, - awxUsername: AWX_E2E_USERNAME, - awxPassword: AWX_E2E_PASSWORD, - asyncHookTimeout: AWX_E2E_TIMEOUT_ASYNC, - longTmeout: AWX_E2E_TIMEOUT_LONG, - mediumTimeout: AWX_E2E_TIMEOUT_MEDIUM, - retryAssertionTimeout: AWX_E2E_TIMEOUT_MEDIUM, - selenium_host: AWX_E2E_SELENIUM_HOST, - selenium_port: AWX_E2E_SELENIUM_PORT, - launch_url: AWX_E2E_LAUNCH_URL, - shortTimeout: AWX_E2E_TIMEOUT_SHORT, - waitForConditionTimeout: AWX_E2E_TIMEOUT_MEDIUM, - test_workers: { - enabled: (AWX_E2E_WORKERS > 0), - workers: AWX_E2E_WORKERS - } + AWX_E2E_CLUSTER_HOST, + AWX_E2E_CLUSTER_PORT, + AWX_E2E_CLUSTER_WORKERS, + AWX_E2E_LAUNCH_URL, + AWX_E2E_PASSWORD, + AWX_E2E_URL, + AWX_E2E_USERNAME, + AWX_E2E_TIMEOUT_ASYNC, + AWX_E2E_TIMEOUT_LONG, + AWX_E2E_TIMEOUT_MEDIUM, + AWX_E2E_TIMEOUT_SHORT }; diff --git a/awx/ui/test/e2e/tests/test-credentials-add-edit-gce.js b/awx/ui/test/e2e/tests/test-credentials-add-edit-gce.js index f0f59f5ba9..a7d64ffdba 100644 --- a/awx/ui/test/e2e/tests/test-credentials-add-edit-gce.js +++ b/awx/ui/test/e2e/tests/test-credentials-add-edit-gce.js @@ -1,5 +1,7 @@ import uuid from 'uuid'; +import { AWX_E2E_TIMEOUT_LONG } from '../settings'; + const testID = uuid().substr(0, 8); const store = { @@ -161,7 +163,7 @@ module.exports = { const row = '#credentials_table tbody tr'; credentials.section.list.section.search - .waitForElementVisible('@input', client.globals.longWait) + .waitForElementVisible('@input', AWX_E2E_TIMEOUT_LONG) .setValue('@input', `name:${store.credential.name}`) .click('@searchButton'); diff --git a/awx/ui/test/e2e/tests/test-credentials-add-edit-insights.js b/awx/ui/test/e2e/tests/test-credentials-add-edit-insights.js index a58457b1d7..b85e0a9592 100644 --- a/awx/ui/test/e2e/tests/test-credentials-add-edit-insights.js +++ b/awx/ui/test/e2e/tests/test-credentials-add-edit-insights.js @@ -1,5 +1,7 @@ import uuid from 'uuid'; +import { AWX_E2E_TIMEOUT_LONG } from '../settings'; + const testID = uuid().substr(0, 8); const store = { @@ -123,7 +125,7 @@ module.exports = { const row = '#credentials_table tbody tr'; credentials.section.list.section.search - .waitForElementVisible('@input', client.globals.longWait) + .waitForElementVisible('@input', AWX_E2E_TIMEOUT_LONG) .setValue('@input', `name:${store.credential.name}`) .click('@searchButton'); diff --git a/awx/ui/test/e2e/tests/test-credentials-add-edit-machine.js b/awx/ui/test/e2e/tests/test-credentials-add-edit-machine.js index 2c1ed44f6d..c16a76a0a5 100644 --- a/awx/ui/test/e2e/tests/test-credentials-add-edit-machine.js +++ b/awx/ui/test/e2e/tests/test-credentials-add-edit-machine.js @@ -1,5 +1,7 @@ import uuid from 'uuid'; +import { AWX_E2E_TIMEOUT_LONG } from '../settings'; + const testID = uuid().substr(0, 8); const store = { @@ -169,7 +171,7 @@ module.exports = { const row = '#credentials_table tbody tr'; credentials.section.list.section.search - .waitForElementVisible('@input', client.globals.longWait) + .waitForElementVisible('@input', AWX_E2E_TIMEOUT_LONG) .setValue('@input', `name:${store.credential.name}`) .click('@searchButton'); diff --git a/awx/ui/test/e2e/tests/test-credentials-add-edit-network.js b/awx/ui/test/e2e/tests/test-credentials-add-edit-network.js index e2cde4e2ce..6b025924b4 100644 --- a/awx/ui/test/e2e/tests/test-credentials-add-edit-network.js +++ b/awx/ui/test/e2e/tests/test-credentials-add-edit-network.js @@ -1,5 +1,7 @@ import uuid from 'uuid'; +import { AWX_E2E_TIMEOUT_LONG } from '../settings'; + const testID = uuid().substr(0, 8); const store = { @@ -194,7 +196,7 @@ module.exports = { const row = '#credentials_table tbody tr'; credentials.section.list.section.search - .waitForElementVisible('@input', client.globals.longWait) + .waitForElementVisible('@input', AWX_E2E_TIMEOUT_LONG) .setValue('@input', `name:${store.credential.name}`) .click('@searchButton'); diff --git a/awx/ui/test/e2e/tests/test-credentials-add-edit-scm.js b/awx/ui/test/e2e/tests/test-credentials-add-edit-scm.js index 9231907afc..8a452bbc70 100644 --- a/awx/ui/test/e2e/tests/test-credentials-add-edit-scm.js +++ b/awx/ui/test/e2e/tests/test-credentials-add-edit-scm.js @@ -1,5 +1,7 @@ import uuid from 'uuid'; +import { AWX_E2E_TIMEOUT_LONG } from '../settings'; + const testID = uuid().substr(0, 8); const store = { @@ -163,7 +165,7 @@ module.exports = { const row = '#credentials_table tbody tr'; credentials.section.list.section.search - .waitForElementVisible('@input', client.globals.longWait) + .waitForElementVisible('@input', AWX_E2E_TIMEOUT_LONG) .setValue('@input', `name:${store.credential.name}`) .click('@searchButton'); diff --git a/awx/ui/test/e2e/tests/test-credentials-add-edit-vault.js b/awx/ui/test/e2e/tests/test-credentials-add-edit-vault.js index 9987666e0f..ad8d38fff8 100644 --- a/awx/ui/test/e2e/tests/test-credentials-add-edit-vault.js +++ b/awx/ui/test/e2e/tests/test-credentials-add-edit-vault.js @@ -1,6 +1,9 @@ import uuid from 'uuid'; +import { AWX_E2E_TIMEOUT_LONG } from '../settings'; + const testID = uuid().substr(0, 8); + const store = { organization: { name: `org-${testID}` @@ -123,7 +126,7 @@ module.exports = { const row = '#credentials_table tbody tr'; credentials.section.list.section.search - .waitForElementVisible('@input', client.globals.longWait) + .waitForElementVisible('@input', AWX_E2E_TIMEOUT_LONG) .setValue('@input', `name:${store.credential.name}`) .click('@searchButton');