libretic/etherpad-lite
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

frontend test improvements (#4161)

Browse source 
* update sauce connect proxy to 4.6.2

* include tunnelIdentifier in webdriver capabilities

* add platform in console output

* include extendedDebugging in webdriver capabilities to get browser console logs

* informative: add comment for timeouts during tests

* When the killTimeout in runner.js stops the tests, it's an failure.

* do not wait a hardcoded amount of 10 seconds for files to be minified.
this setup time is not included in the total time of the first test.

* run 4 browsers at a time during frontend testing

* try to include test.speed in output

* time is in test.duration, not test.speed

* frontend tests: 6 sessions in parallel, add OSX 10.14-safari and Windows7-firefox, pin all browsers instead of use latest

* typo
This commit is contained in:
webzwo0i 2020-07-13 16:12:39 +02:00 committed by GitHub
parent 3bd81e6d01
commit 2c8fb99be8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 36 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
tests/frontend/runner.js
View file

@ -60,7 +60,7 @@ $(function(){
stats.tests = stats.tests || 0;
stats.tests++;
if ('passed' == test.state) {
append("->","[green]PASSED[clear] :", test.title);
append("->","[green]PASSED[clear] :", test.title," ",test.duration,"ms");
} else if (test.pending) {
append("->","[yellow]PENDING[clear]:", test.title);
} else {

47
tests/frontend/travis/remote_runner.js
View file

@ -17,9 +17,9 @@ var sauceTestWorker = async.queue(function (testSettings, callback) {
testSettings.name = name;
testSettings["public"] = true;
testSettings["build"] = process.env.GIT_HASH;
testSettings["extendedDebugging"] = true; // console.json can be downloaded via saucelabs, don't know how to print them into output of the tests
testSettings["tunnelIdentifier"] = process.env.TRAVIS_JOB_NUMBER;
// we wait 10 seconds here with the hope it was enough time for the minified files to be built etc.
setTimeout(function(){
browser.init(testSettings).get("http://localhost:9001/tests/frontend/", function(){
var url = "https://saucelabs.com/jobs/" + browser.sessionID;
console.log("Remote sauce test '" + name + "' started! " + url);
@ -38,7 +38,7 @@ var sauceTestWorker = async.queue(function (testSettings, callback) {
var testResult = knownConsoleText.replace(/\[red\]/g,'\x1B[31m').replace(/\[yellow\]/g,'\x1B[33m')
.replace(/\[green\]/g,'\x1B[32m').replace(/\[clear\]/g, '\x1B[39m');
testResult = testResult.split("\\n").map(function(line){
return "[" + testSettings.browserName + (testSettings.version === "" ? '' : (" " + testSettings.version)) + "] " + line;
return "[" + testSettings.browserName + " " + testSettings.platform + (testSettings.version === "" ? '' : (" " + testSettings.version)) + "] " + line;
}).join("\n");
console.log(testResult);
@ -47,7 +47,10 @@ var sauceTestWorker = async.queue(function (testSettings, callback) {
callback();
}
//timeout for the case the test hangs
/**
* timeout for the case the test hangs
* @todo this should be configured in testSettings, see https://wiki.saucelabs.com/display/DOCS/Test+Configuration+Options#TestConfigurationOptions-Timeouts
*/
var timeout = setTimeout(function(){
stopSauce(false);
}, 1200000 * 10);
@ -61,36 +64,46 @@ var sauceTestWorker = async.queue(function (testSettings, callback) {
knownConsoleText = consoleText;
if(knownConsoleText.indexOf("FINISHED") > 0){
var success = knownConsoleText.indexOf("FAILED") === -1;
stopSauce(success);
let match = knownConsoleText.match(/FINISHED - ([0-9]+) tests passed, ([0-9]+) tests failed/);
if (match[2] && match[2] == 0){
stopSauce(true);
}
else {
stopSauce(false);
}
}
});
}, 5000);
});
}, 10000);
}, 1); //run 1 test in parrallel
}, 6); //run 6 tests in parrallel
// 1) Firefox on Linux
sauceTestWorker.push({
'platform' : 'Linux'
, 'browserName' : 'firefox'
, 'version' : 'latest'
, 'version' : '45.0'
});
// 2) Chrome on Linux
sauceTestWorker.push({
'platform' : 'Linux'
, 'browserName' : 'googlechrome'
, 'version' : 'latest'
, 'browserName' : 'chrome'
, 'version' : '48.0'
});
// 3) Safari on OSX 10.15
sauceTestWorker.push({
'platform' : 'OS X 10.15'
, 'browserName' : 'safari'
, 'version' : 'latest'
, 'version' : '13.1'
});
// 4) Safari on OSX 10.14
sauceTestWorker.push({
'platform' : 'OS X 10.14'
, 'browserName' : 'safari'
, 'version' : '12.0'
});
// IE 10 doesn't appear to be working anyway
/*
@ -105,7 +118,13 @@ sauceTestWorker.push({
sauceTestWorker.push({
'platform' : 'Windows 10'
, 'browserName' : 'microsoftedge'
, 'version' : 'latest'
, 'version' : '83.0'
});
// 6) Firefox on Win 7
sauceTestWorker.push({
'platform' : 'Windows 7'
, 'browserName' : 'firefox'
, 'version' : '78.0'
});
sauceTestWorker.drain = function() {

4
tests/frontend/travis/sauce_tunnel.sh
View file

@ -9,12 +9,12 @@
if [ -z "${SAUCE_USERNAME}" ]; then echo "SAUCE_USERNAME is unset - exiting"; exit 1; fi
if [ -z "${SAUCE_ACCESS_KEY}" ]; then echo "SAUCE_ACCESS_KEY is unset - exiting"; exit 1; fi
curl https://saucelabs.com/downloads/sc-4.5.4-linux.tar.gz > /tmp/sauce.tar.gz
curl https://saucelabs.com/downloads/sc-4.6.2-linux.tar.gz > /tmp/sauce.tar.gz
tar zxf /tmp/sauce.tar.gz --directory /tmp
mv /tmp/sc-*-linux /tmp/sauce_connect
# start the sauce connector in background and make sure it doesn't output the secret key
(/tmp/sauce_connect/bin/sc --user "${SAUCE_USERNAME}" --key "${SAUCE_ACCESS_KEY}" --pidfile /tmp/sauce.pid --readyfile /tmp/tunnel > /dev/null )&
(/tmp/sauce_connect/bin/sc --user "${SAUCE_USERNAME}" --key "${SAUCE_ACCESS_KEY}" -i "${TRAVIS_JOB_NUMBER}" --pidfile /tmp/sauce.pid --readyfile /tmp/tunnel > /dev/null )&
# wait for the tunnel to build up
while [ ! -e "/tmp/tunnel" ]