From 2c8fb99be8de44754b405203ff3b7f336401ef33 Mon Sep 17 00:00:00 2001 From: webzwo0i Date: Mon, 13 Jul 2020 16:12:39 +0200 Subject: [PATCH] frontend test improvements (#4161) * update sauce connect proxy to 4.6.2 * include tunnelIdentifier in webdriver capabilities * add platform in console output * include extendedDebugging in webdriver capabilities to get browser console logs * informative: add comment for timeouts during tests * When the killTimeout in runner.js stops the tests, it's an failure. * do not wait a hardcoded amount of 10 seconds for files to be minified. this setup time is not included in the total time of the first test. * run 4 browsers at a time during frontend testing * try to include test.speed in output * time is in test.duration, not test.speed * frontend tests: 6 sessions in parallel, add OSX 10.14-safari and Windows7-firefox, pin all browsers instead of use latest * typo --- tests/frontend/runner.js | 2 +- tests/frontend/travis/remote_runner.js | 47 ++++++++++++++++++-------- tests/frontend/travis/sauce_tunnel.sh | 4 +-- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/tests/frontend/runner.js b/tests/frontend/runner.js index f233e7a9..e9eedc64 100644 --- a/tests/frontend/runner.js +++ b/tests/frontend/runner.js @@ -60,7 +60,7 @@ $(function(){ stats.tests = stats.tests || 0; stats.tests++; if ('passed' == test.state) { - append("->","[green]PASSED[clear] :", test.title); + append("->","[green]PASSED[clear] :", test.title," ",test.duration,"ms"); } else if (test.pending) { append("->","[yellow]PENDING[clear]:", test.title); } else { diff --git a/tests/frontend/travis/remote_runner.js b/tests/frontend/travis/remote_runner.js index 1e04b2a0..48f2aedd 100644 --- a/tests/frontend/travis/remote_runner.js +++ b/tests/frontend/travis/remote_runner.js @@ -17,9 +17,9 @@ var sauceTestWorker = async.queue(function (testSettings, callback) { testSettings.name = name; testSettings["public"] = true; testSettings["build"] = process.env.GIT_HASH; + testSettings["extendedDebugging"] = true; // console.json can be downloaded via saucelabs, don't know how to print them into output of the tests + testSettings["tunnelIdentifier"] = process.env.TRAVIS_JOB_NUMBER; - // we wait 10 seconds here with the hope it was enough time for the minified files to be built etc. - setTimeout(function(){ browser.init(testSettings).get("http://localhost:9001/tests/frontend/", function(){ var url = "https://saucelabs.com/jobs/" + browser.sessionID; console.log("Remote sauce test '" + name + "' started! " + url); @@ -38,7 +38,7 @@ var sauceTestWorker = async.queue(function (testSettings, callback) { var testResult = knownConsoleText.replace(/\[red\]/g,'\x1B[31m').replace(/\[yellow\]/g,'\x1B[33m') .replace(/\[green\]/g,'\x1B[32m').replace(/\[clear\]/g, '\x1B[39m'); testResult = testResult.split("\\n").map(function(line){ - return "[" + testSettings.browserName + (testSettings.version === "" ? '' : (" " + testSettings.version)) + "] " + line; + return "[" + testSettings.browserName + " " + testSettings.platform + (testSettings.version === "" ? '' : (" " + testSettings.version)) + "] " + line; }).join("\n"); console.log(testResult); @@ -47,7 +47,10 @@ var sauceTestWorker = async.queue(function (testSettings, callback) { callback(); } - //timeout for the case the test hangs + /** + * timeout for the case the test hangs + * @todo this should be configured in testSettings, see https://wiki.saucelabs.com/display/DOCS/Test+Configuration+Options#TestConfigurationOptions-Timeouts + */ var timeout = setTimeout(function(){ stopSauce(false); }, 1200000 * 10); @@ -61,36 +64,46 @@ var sauceTestWorker = async.queue(function (testSettings, callback) { knownConsoleText = consoleText; if(knownConsoleText.indexOf("FINISHED") > 0){ - var success = knownConsoleText.indexOf("FAILED") === -1; - stopSauce(success); + let match = knownConsoleText.match(/FINISHED - ([0-9]+) tests passed, ([0-9]+) tests failed/); + if (match[2] && match[2] == 0){ + stopSauce(true); + } + else { + stopSauce(false); + } } }); }, 5000); }); - }, 10000); - -}, 1); //run 1 test in parrallel +}, 6); //run 6 tests in parrallel // 1) Firefox on Linux sauceTestWorker.push({ 'platform' : 'Linux' , 'browserName' : 'firefox' - , 'version' : 'latest' + , 'version' : '45.0' }); // 2) Chrome on Linux sauceTestWorker.push({ 'platform' : 'Linux' - , 'browserName' : 'googlechrome' - , 'version' : 'latest' + , 'browserName' : 'chrome' + , 'version' : '48.0' }); // 3) Safari on OSX 10.15 sauceTestWorker.push({ 'platform' : 'OS X 10.15' , 'browserName' : 'safari' - , 'version' : 'latest' + , 'version' : '13.1' +}); + +// 4) Safari on OSX 10.14 +sauceTestWorker.push({ + 'platform' : 'OS X 10.14' + , 'browserName' : 'safari' + , 'version' : '12.0' }); // IE 10 doesn't appear to be working anyway /* @@ -105,7 +118,13 @@ sauceTestWorker.push({ sauceTestWorker.push({ 'platform' : 'Windows 10' , 'browserName' : 'microsoftedge' - , 'version' : 'latest' + , 'version' : '83.0' +}); +// 6) Firefox on Win 7 +sauceTestWorker.push({ + 'platform' : 'Windows 7' + , 'browserName' : 'firefox' + , 'version' : '78.0' }); sauceTestWorker.drain = function() { diff --git a/tests/frontend/travis/sauce_tunnel.sh b/tests/frontend/travis/sauce_tunnel.sh index b8e1bc59..4ab6e81a 100755 --- a/tests/frontend/travis/sauce_tunnel.sh +++ b/tests/frontend/travis/sauce_tunnel.sh @@ -9,12 +9,12 @@ if [ -z "${SAUCE_USERNAME}" ]; then echo "SAUCE_USERNAME is unset - exiting"; exit 1; fi if [ -z "${SAUCE_ACCESS_KEY}" ]; then echo "SAUCE_ACCESS_KEY is unset - exiting"; exit 1; fi -curl https://saucelabs.com/downloads/sc-4.5.4-linux.tar.gz > /tmp/sauce.tar.gz +curl https://saucelabs.com/downloads/sc-4.6.2-linux.tar.gz > /tmp/sauce.tar.gz tar zxf /tmp/sauce.tar.gz --directory /tmp mv /tmp/sc-*-linux /tmp/sauce_connect # start the sauce connector in background and make sure it doesn't output the secret key -(/tmp/sauce_connect/bin/sc --user "${SAUCE_USERNAME}" --key "${SAUCE_ACCESS_KEY}" --pidfile /tmp/sauce.pid --readyfile /tmp/tunnel > /dev/null )& +(/tmp/sauce_connect/bin/sc --user "${SAUCE_USERNAME}" --key "${SAUCE_ACCESS_KEY}" -i "${TRAVIS_JOB_NUMBER}" --pidfile /tmp/sauce.pid --readyfile /tmp/tunnel > /dev/null )& # wait for the tunnel to build up while [ ! -e "/tmp/tunnel" ]