From 1c0e9d8015309d61359e7666e1468c7f5fb4aa64 Mon Sep 17 00:00:00 2001 From: Otto Richter Date: Sat, 21 Jun 2025 11:50:39 +0200 Subject: [PATCH] chore(ci): downgrade playwright temporarily and allow running all e2e tests (#8245) In https://codeberg.org/forgejo/forgejo/pulls/7906#issuecomment-5511884, I noticed that the e2e tests were failing without obvious reasons. I was able to reproduce locally with Mobile Chrome, but the error doesn't make sense to me. So I tried to downgrade playwright to the previous version, and it works fine. I actually suspect a bug in playwright, but I currently lack the capacity to reach out to upstream with a reproducer (I also found conversation with the playwright team a little difficult, unless you have absolutely convincing arguments that the flakiness you observe is really their fault, which is very hard to prove). Tests pass with this version of playwright. In order to detect such cases earlier, I added a way to run all playwright tests (which I thought I had added with the changed files patch, but apparently forgot). All tests are triggered by an explicit label (but only after firing a next event, because the testing workflows don't listen to label changes) and when the PR title contains "playwright", which should cover at least renovate dependency updates of playwright and the axe framework (both contain playwright). Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/8245 Reviewed-by: Earl Warren Reviewed-by: Michael Kriese Reviewed-by: Beowulf Co-authored-by: Otto Richter Co-committed-by: Otto Richter --- .forgejo/workflows/testing.yml | 6 ++++++ package-lock.json | 24 ++++++++++++------------ package.json | 2 +- tests/e2e/changes.go | 9 +++++++-- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml index 86e74591f1..7a93bb66a8 100644 --- a/.forgejo/workflows/testing.yml +++ b/.forgejo/workflows/testing.yml @@ -115,6 +115,11 @@ jobs: run: | su forgejo -c 'make deps-frontend frontend' - uses: ./.forgejo/workflows-composite/build-backend + - name: Decide to run all tests + id: run-all + if: contains(github.event.pull_request.labels.*.name, 'run-all-playwright-tests') || contains(github.event.pull_request.title, 'playwright') + run: | + echo "all=1" >> "$GITHUB_OUTPUT" - name: Get changed files id: changed-files uses: https://data.forgejo.org/tj-actions/changed-files@v46 @@ -127,6 +132,7 @@ jobs: USE_REPO_TEST_DIR: 1 PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 CHANGED_FILES: ${{steps.changed-files.outputs.all_changed_files}} + RUN_ALL: ${{steps.run-all.all}} - name: Upload test artifacts on failure if: failure() uses: https://data.forgejo.org/forgejo/upload-artifact@v4 diff --git a/package-lock.json b/package-lock.json index 179c0e496c..1637629a83 100644 --- a/package-lock.json +++ b/package-lock.json @@ -62,7 +62,7 @@ "devDependencies": { "@axe-core/playwright": "4.10.2", "@eslint-community/eslint-plugin-eslint-comments": "4.5.0", - "@playwright/test": "1.53.0", + "@playwright/test": "1.52.0", "@stoplight/spectral-cli": "6.15.0", "@stylistic/eslint-plugin": "4.4.1", "@stylistic/stylelint-plugin": "3.1.2", @@ -2143,13 +2143,13 @@ } }, "node_modules/@playwright/test": { - "version": "1.53.0", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.53.0.tgz", - "integrity": "sha512-15hjKreZDcp7t6TL/7jkAo6Df5STZN09jGiv5dbP9A6vMVncXRqE7/B2SncsyOwrkZRBH2i6/TPOL8BVmm3c7w==", + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.52.0.tgz", + "integrity": "sha512-uh6W7sb55hl7D6vsAeA+V2p5JnlAqzhqFyF0VcJkKZXkgnFcVG9PziERRHQfPLfNGx1C292a4JqbWzhR8L4R1g==", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright": "1.53.0" + "playwright": "1.52.0" }, "bin": { "playwright": "cli.js" @@ -11859,13 +11859,13 @@ } }, "node_modules/playwright": { - "version": "1.53.0", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.53.0.tgz", - "integrity": "sha512-ghGNnIEYZC4E+YtclRn4/p6oYbdPiASELBIYkBXfaTVKreQUYbMUYQDwS12a8F0/HtIjr/CkGjtwABeFPGcS4Q==", + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.52.0.tgz", + "integrity": "sha512-JAwMNMBlxJ2oD1kce4KPtMkDeKGHQstdpFPcPH3maElAXon/QZeTvtsfXmTMRyO9TslfoYOXkSsvao2nE1ilTw==", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.53.0" + "playwright-core": "1.52.0" }, "bin": { "playwright": "cli.js" @@ -11878,9 +11878,9 @@ } }, "node_modules/playwright-core": { - "version": "1.53.0", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.53.0.tgz", - "integrity": "sha512-mGLg8m0pm4+mmtB7M89Xw/GSqoNC+twivl8ITteqvAndachozYe2ZA7srU6uleV1vEdAHYqjq+SV8SNxRRFYBw==", + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.52.0.tgz", + "integrity": "sha512-l2osTgLXSMeuLZOML9qYODUQoPPnUsKsb5/P6LJ2e6uPKXUdPK5WYhN4z03G+YNbWmGDY4YENauNu4ZKczreHg==", "dev": true, "license": "Apache-2.0", "bin": { diff --git a/package.json b/package.json index 409edecd95..d7c3a5f79f 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "devDependencies": { "@axe-core/playwright": "4.10.2", "@eslint-community/eslint-plugin-eslint-comments": "4.5.0", - "@playwright/test": "1.53.0", + "@playwright/test": "1.52.0", "@stoplight/spectral-cli": "6.15.0", "@stylistic/eslint-plugin": "4.4.1", "@stylistic/stylelint-plugin": "3.1.2", diff --git a/tests/e2e/changes.go b/tests/e2e/changes.go index d1d318fd06..5e9238dfa7 100644 --- a/tests/e2e/changes.go +++ b/tests/e2e/changes.go @@ -16,10 +16,15 @@ import ( var ( changesetFiles []string changesetAvailable bool - globalFullRun bool + globalFullRun = false ) func initChangedFiles() { + _, globalFullRun = os.LookupEnv("RUN_ALL") + if globalFullRun { + log.Info("Full run of all tests requested via RUN_ALL environment.") + return + } var changes string changes, changesetAvailable = os.LookupEnv("CHANGED_FILES") // the output of the Action seems to actually contain \n and not a newline literal @@ -44,7 +49,7 @@ func initChangedFiles() { for _, expr := range globalPatterns { fullRunPatterns = append(fullRunPatterns, glob.MustCompile(expr, '.', '/')) } - globalFullRun = false + for _, changedFile := range changesetFiles { for _, pattern := range fullRunPatterns { if pattern.Match(changedFile) {