From b71b60677458049b4d8e77bc57407e35f5782450 Mon Sep 17 00:00:00 2001 From: webzwo0i Date: Tue, 24 Nov 2020 19:12:41 +0100 Subject: [PATCH] tests: Switch from Travis to Github Actions Travis placed an unnecessary breaking restriction on our tests and failed to respond within 72 hours to our complaint. This has forced us to introduce Github Actions to manage our testing. This is hopefully a temporary measure while Travis either gets itself together or we find a non-Github requirement. --- .github/workflows/backend-tests.yml | 60 ++++++++++++++++++ .github/workflows/dockerfile.yml | 26 ++++++++ .github/workflows/frontend-tests.yml | 83 +++++++++++++++++++++++++ .github/workflows/lint-package-lock.yml | 24 +++++++ .github/workflows/load-test.yml | 53 ++++++++++++++++ .github/workflows/rate-limit.yml | 39 ++++++++++++ tests/frontend/travis/remote_runner.js | 7 ++- tests/ratelimit/testlimits.sh | 0 8 files changed, 291 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/backend-tests.yml create mode 100644 .github/workflows/dockerfile.yml create mode 100644 .github/workflows/frontend-tests.yml create mode 100644 .github/workflows/lint-package-lock.yml create mode 100644 .github/workflows/load-test.yml create mode 100644 .github/workflows/rate-limit.yml mode change 100644 => 100755 tests/ratelimit/testlimits.sh diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml new file mode 100644 index 00000000..e50491d2 --- /dev/null +++ b/.github/workflows/backend-tests.yml @@ -0,0 +1,60 @@ +name: "Backend tests" + +# any branch is useful for testing before a PR is submitted +on: [push, pull_request] + +jobs: + withoutplugins: + # run on pushes to any branch + # run on PRs from external forks + if: | + (github.event_name != 'pull_request') + || (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id) + name: without plugins + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install libreoffice + run: | + sudo add-apt-repository -y ppa:libreoffice/ppa + sudo apt update + sudo apt install -y --no-install-recommends libreoffice libreoffice-pdfimport + + - name: Install all dependencies and symlink for ep_etherpad-lite + run: bin/installDeps.sh + + # configures some settings and runs npm run test + - name: Run the backend tests + run: tests/frontend/travis/runnerBackend.sh + + withplugins: + # run on pushes to any branch + # run on PRs from external forks + if: | + (github.event_name != 'pull_request') + || (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id) + name: with Plugins + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install libreoffice + run: | + sudo add-apt-repository -y ppa:libreoffice/ppa + sudo apt update + sudo apt install -y --no-install-recommends libreoffice libreoffice-pdfimport + + - name: Install all dependencies and symlink for ep_etherpad-lite + run: bin/installDeps.sh + + - name: Install etherpad plugins + run: npm install ep_align ep_author_hover ep_cursortrace ep_font_size ep_hash_auth ep_headings2 ep_markdown ep_readonly_guest ep_spellcheck ep_subscript_and_superscript ep_table_of_contents + + # configures some settings and runs npm run test + - name: Run the backend tests + run: tests/frontend/travis/runnerBackend.sh diff --git a/.github/workflows/dockerfile.yml b/.github/workflows/dockerfile.yml new file mode 100644 index 00000000..8f6d5c3b --- /dev/null +++ b/.github/workflows/dockerfile.yml @@ -0,0 +1,26 @@ +name: "Dockerfile" + +# any branch is useful for testing before a PR is submitted +on: [push, pull_request] + +jobs: + dockerfile: + # run on pushes to any branch + # run on PRs from external forks + if: | + (github.event_name != 'pull_request') + || (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id) + name: build image and run connectivity test + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: docker build + run: | + docker build -t etherpad:test . + docker run -d -p 9001:9001 etherpad:test + ./bin/installDeps.sh + sleep 3 # delay for startup? + cd src && npm run test-container diff --git a/.github/workflows/frontend-tests.yml b/.github/workflows/frontend-tests.yml new file mode 100644 index 00000000..7a2846db --- /dev/null +++ b/.github/workflows/frontend-tests.yml @@ -0,0 +1,83 @@ +name: "Frontend tests" + +on: [push] + +jobs: + withoutplugins: + name: without plugins + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Run sauce-connect-action + shell: bash + env: + SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} + SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} + TRAVIS_JOB_NUMBER: ${{ github.run_id }}-${{ github.run_number }}-${{ github.job }} + run: tests/frontend/travis/sauce_tunnel.sh + + - name: Install all dependencies and symlink for ep_etherpad-lite + run: bin/installDeps.sh + + - name: export GIT_HASH to env + id: environment + run: echo "::set-output name=sha_short::$(git rev-parse --short ${{ github.sha }})" + + - name: Write custom settings.json with loglevel WARN + run: "sed 's/\"loglevel\": \"INFO\",/\"loglevel\": \"WARN\",/' < settings.json.template > settings.json" + + - name: Run the frontend tests + shell: bash + env: + SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} + SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} + TRAVIS_JOB_NUMBER: ${{ github.run_id }}-${{ github.run_number }}-${{ github.job }} + GIT_HASH: ${{ steps.environment.outputs.sha_short }} + run: | + tests/frontend/travis/runner.sh + + withplugins: + name: with plugins + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Run sauce-connect-action + shell: bash + env: + SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} + SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} + TRAVIS_JOB_NUMBER: ${{ github.run_id }}-${{ github.run_number }}-${{ github.job }} + run: tests/frontend/travis/sauce_tunnel.sh + + - name: Install all dependencies and symlink for ep_etherpad-lite + run: bin/installDeps.sh + + - name: Install etherpad plugins + run: npm install ep_align ep_author_hover ep_cursortrace ep_font_size ep_hash_auth ep_headings2 ep_markdown ep_readonly_guest ep_spellcheck ep_subscript_and_superscript ep_table_of_contents + + - name: export GIT_HASH to env + id: environment + run: echo "::set-output name=sha_short::$(git rev-parse --short ${{ github.sha }})" + + - name: Write custom settings.json with loglevel WARN + run: "sed 's/\"loglevel\": \"INFO\",/\"loglevel\": \"WARN\",/' < settings.json.template > settings.json" + + # XXX we should probably run all tests, because plugins could effect their results + - name: Remove standard frontend test files, so only plugin tests are run + run: rm tests/frontend/specs/* + + - name: Run the frontend tests + shell: bash + env: + SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} + SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} + TRAVIS_JOB_NUMBER: ${{ github.run_id }}-${{ github.run_number }}-${{ github.job }} + GIT_HASH: ${{ steps.environment.outputs.sha_short }} + run: | + tests/frontend/travis/runner.sh diff --git a/.github/workflows/lint-package-lock.yml b/.github/workflows/lint-package-lock.yml new file mode 100644 index 00000000..beef64ff --- /dev/null +++ b/.github/workflows/lint-package-lock.yml @@ -0,0 +1,24 @@ +name: "Lint" + +# any branch is useful for testing before a PR is submitted +on: [push, pull_request] + +jobs: + lint-package-lock: + # run on pushes to any branch + # run on PRs from external forks + if: | + (github.event_name != 'pull_request') + || (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id) + name: package-lock.json + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install lockfile-lint + run: npm install lockfile-lint + + - name: Run lockfile-lint on package-lock.json + run: npx lockfile-lint --path src/package-lock.json --validate-https --allowed-hosts npm diff --git a/.github/workflows/load-test.yml b/.github/workflows/load-test.yml new file mode 100644 index 00000000..095adc78 --- /dev/null +++ b/.github/workflows/load-test.yml @@ -0,0 +1,53 @@ +name: "Loadtest" + +# any branch is useful for testing before a PR is submitted +on: [push, pull_request] + +jobs: + withoutplugins: + # run on pushes to any branch + # run on PRs from external forks + if: | + (github.event_name != 'pull_request') + || (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id) + name: without plugins + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install all dependencies and symlink for ep_etherpad-lite + run: bin/installDeps.sh + + - name: Install etherpad-load-test + run: sudo npm install -g etherpad-load-test + + - name: Run load test + run: tests/frontend/travis/runnerLoadTest.sh + + withplugins: + # run on pushes to any branch + # run on PRs from external forks + if: | + (github.event_name != 'pull_request') + || (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id) + name: with Plugins + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install all dependencies and symlink for ep_etherpad-lite + run: bin/installDeps.sh + + - name: Install etherpad-load-test + run: sudo npm install -g etherpad-load-test + + - name: Install etherpad plugins + run: npm install ep_align ep_author_hover ep_cursortrace ep_font_size ep_hash_auth ep_headings2 ep_markdown ep_readonly_guest ep_spellcheck ep_subscript_and_superscript ep_table_of_contents + + # configures some settings and runs npm run test + - name: Run load test + run: tests/frontend/travis/runnerLoadTest.sh diff --git a/.github/workflows/rate-limit.yml b/.github/workflows/rate-limit.yml new file mode 100644 index 00000000..4bdfc219 --- /dev/null +++ b/.github/workflows/rate-limit.yml @@ -0,0 +1,39 @@ +name: "rate limit" + +# any branch is useful for testing before a PR is submitted +on: [push, pull_request] + +jobs: + ratelimit: + # run on pushes to any branch + # run on PRs from external forks + if: | + (github.event_name != 'pull_request') + || (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id) + name: test + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: docker network + run: docker network create --subnet=172.23.42.0/16 ep_net + + - name: build docker image + run: | + docker build -f Dockerfile -t epl-debian-slim . + docker build -f tests/ratelimit/Dockerfile.nginx -t nginx-latest . + docker build -f tests/ratelimit/Dockerfile.anotherip -t anotherip . + - name: run docker images + run: | + docker run --name etherpad-docker -p 9000:9001 --rm --network ep_net --ip 172.23.42.2 -e 'TRUST_PROXY=true' epl-debian-slim & + docker run -p 8081:80 --rm --network ep_net --ip 172.23.42.1 -d nginx-latest + docker run --rm --network ep_net --ip 172.23.42.3 --name anotherip -dt anotherip + + - name: install dependencies and create symlink for ep_etherpad-lite + run: bin/installDeps.sh + + - name: run rate limit test + run: | + cd tests/ratelimit + ./testlimits.sh diff --git a/tests/frontend/travis/remote_runner.js b/tests/frontend/travis/remote_runner.js index 40e01e78..0907b7c2 100644 --- a/tests/frontend/travis/remote_runner.js +++ b/tests/frontend/travis/remote_runner.js @@ -13,7 +13,12 @@ var allTestsPassed = true; // overwrite the default exit code // in case not all worker can be run (due to saucelabs limits), `queue.drain` below will not be called // and the script would silently exit with error code 0 -process.exitCode = 1; +process.exitCode = 2; +process.on('exit', (code) => { + if (code === 2){ + console.log("\x1B[31mFAILED\x1B[39m Not all saucelabs runner have been started."); + } +}) var sauceTestWorker = async.queue(function (testSettings, callback) { var browser = wd.promiseChainRemote(config.host, config.port, config.username, config.accessKey); diff --git a/tests/ratelimit/testlimits.sh b/tests/ratelimit/testlimits.sh old mode 100644 new mode 100755