Issue 1:
System info

Playwright Version: v1.31.0

Operating System: Ubuntu 22.04.2

Browser: Firefox

Source code

import com.microsoft.playwright.BrowserType;
import com.microsoft.playwright.Playwright;
import static com.microsoft.playwright.assertions.PlaywrightAssertions.assertThat;
public class PlaywrightBug {
public static void main(String[] args) throws Exception {
var proc = new ProcessBuilder("docker", "run", "--rm", "-p", "127.0.0.1:80:8080", "ghcr.io/jglick/authenticated-jenkins:main").inheritIO().start();
try (
var playwright = Playwright.create();
var browser = playwright.firefox().launch(new BrowserType.LaunchOptions().setHeadless(false));
var context = browser.newContext();
var page = context.newPage()) {
while (!page.navigate("http://jenkins.127.0.0.1.nip.io/login").ok()) {
System.err.println("Waiting for Jenkins to become ready…");
Thread.sleep(1000);
}
page.getByPlaceholder("Username").fill("admin");
page.getByPlaceholder("Password").fill("admin");
// Does not work; label intercepts click: page.getByLabel("Keep me signed in").check()
page.getByText("Keep me signed in").click();
page.getByText("Sign in").click();
assertThat(page.getByText("Welcome to Jenkins!")).isVisible();
} finally {
proc.destroy();
}
System.exit(0);
}
}

Steps

Run the program.

Expected

Passes.

Actual

Sometimes passes. Sometimes Firefox opens a tab showing https://support.mozilla.org/en-US/kb/insecure-password-warning-firefox?as=u&utm_source=inproduct which blocks visibility of the login form and prevents the test from proceeding:

Exception in thread "main" com.microsoft.playwright.TimeoutError: Error {
message='Timeout 30000ms exceeded.
=========================== logs ===========================
waiting for getByText("Keep me signed in")
locator resolved to <label for="remember_me">Keep me signed in</label>
attempting click action
waiting for element to be visible, enabled and stable
element is visible, enabled and stable
scrolling into view if needed
done scrolling
performing click action
============================================================
name='TimeoutError
stack='TimeoutError: Timeout 30000ms exceeded.
=========================== logs ===========================
waiting for getByText("Keep me signed in")
locator resolved to <label for="remember_me">Keep me signed in</label>
attempting click action
waiting for element to be visible, enabled and stable
element is visible, enabled and stable
scrolling into view if needed
done scrolling
performing click action
============================================================
at ProgressController.run (/tmp/playwright-java-1866671914204587060/package/lib/server/progress.js:88:26)
at Frame.click (/tmp/playwright-java-1866671914204587060/package/lib/server/frames.js:1011:23)
at FrameDispatcher.click (/tmp/playwright-java-1866671914204587060/package/lib/server/dispatchers/frameDispatcher.js:149:30)
at DispatcherConnection.dispatch (/tmp/playwright-java-1866671914204587060/package/lib/server/dispatchers/dispatcher.js:319:46)
}
at com.microsoft.playwright.impl.WaitableResult.get(WaitableResult.java:52)
at com.microsoft.playwright.impl.ChannelOwner.runUntil(ChannelOwner.java:111)
at com.microsoft.playwright.impl.Connection.sendMessage(Connection.java:119)
at com.microsoft.playwright.impl.ChannelOwner.sendMessage(ChannelOwner.java:102)
at com.microsoft.playwright.impl.FrameImpl.clickImpl(FrameImpl.java:252)
at com.microsoft.playwright.impl.FrameImpl.lambda$click$8(FrameImpl.java:243)
at com.microsoft.playwright.impl.LoggingSupport.lambda$withLogging$0(LoggingSupport.java:36)
at com.microsoft.playwright.impl.LoggingSupport.withLogging(LoggingSupport.java:47)
at com.microsoft.playwright.impl.ChannelOwner.withLogging(ChannelOwner.java:87)
at com.microsoft.playwright.impl.LoggingSupport.withLogging(LoggingSupport.java:35)
at com.microsoft.playwright.impl.FrameImpl.click(FrameImpl.java:243)
at com.microsoft.playwright.impl.LocatorImpl.click(LocatorImpl.java:123)
at com.microsoft.playwright.Locator.click(Locator.java:2183)
at PlaywrightBug.main(PlaywrightBug.java:19)
Caused by: com.microsoft.playwright.TimeoutError: Error {
message='Timeout 30000ms exceeded.
=========================== logs ===========================
waiting for getByText("Keep me signed in")
locator resolved to <label for="remember_me">Keep me signed in</label>
attempting click action
waiting for element to be visible, enabled and stable
element is visible, enabled and stable
scrolling into view if needed
done scrolling
performing click action
============================================================
name='TimeoutError
stack='TimeoutError: Timeout 30000ms exceeded.
=========================== logs ===========================
waiting for getByText("Keep me signed in")
locator resolved to <label for="remember_me">Keep me signed in</label>
attempting click action
waiting for element to be visible, enabled and stable
element is visible, enabled and stable
scrolling into view if needed
done scrolling
performing click action
============================================================
at ProgressController.run (/tmp/playwright-java-1866671914204587060/package/lib/server/progress.js:88:26)
at Frame.click (/tmp/playwright-java-1866671914204587060/package/lib/server/frames.js:1011:23)
at FrameDispatcher.click (/tmp/playwright-java-1866671914204587060/package/lib/server/dispatchers/frameDispatcher.js:149:30)
at DispatcherConnection.dispatch (/tmp/playwright-java-1866671914204587060/package/lib/server/dispatchers/dispatcher.js:319:46)
}
at com.microsoft.playwright.impl.Connection.dispatch(Connection.java:199)
at com.microsoft.playwright.impl.Connection.processOneMessage(Connection.java:181)
at com.microsoft.playwright.impl.ChannelOwner.runUntil(ChannelOwner.java:109)
... 12 more

Issue 2:
We have retries set to 1 globally. Occasionally we have a fixture timeout during setup. If this happens on a test that is skipped during runtime with test.skip(), the initial fixture timeout fails the build despite that the retry successfully skipped.

System info

Playwright Version: 1.35

Operating System: Ubuntu 20.04

Browser: Chromium (presumably all)

Other info:

Screenshots

Initial attempt: fixture timeout

Retry: fixture completes, test skipped during runtime

Build failure:

Source code

I provided exact source code that allows reproducing the issue locally.

Config file

// playwright.config.ts
import type { PlaywrightTestConfig } from '@playwright/test';
import { devices } from '@playwright/test';

const config: PlaywrightTestConfig = {
globalSetup: './src/etc/global-setup.ts',
globalTeardown: './src/etc/global-teardown.ts',
testDir: './src/tests',
timeout: 60_000,
expect: {
timeout: 8_000,
},
fullyParallel: true,
forbidOnly: !!process.env.CI,
retries: Number.isNaN(+process.env.RETRIES!) ? (process.env.CI ? 1 : 0) : +process.env.RETRIES!,
workers: 3,
reporter: [
[process.env.CI ? 'dot' : 'line'],
['html', { outputFolder: 'reports/html', open: 'never' }],
['junit', { outputFile: 'reports/junit/junit.xml' }],
],
use: {
actionTimeout: 15_000,
navigationTimeout: 30_000,
baseURL: process.env.ORIGINAL_BASE_URL,
screenshot: 'only-on-failure',
trace: !!process.env.CI ? 'retain-on-failure' : 'on',
video: !!process.env.CI ? 'retain-on-failure' : 'off',
},

projects: [
{
name: 'chromium',
use: {
...devices['Desktop Chrome'],
permissions: ['clipboard-read', 'clipboard-write'],
launchOptions: {
slowMo: +(process.env.SLOWMO ?? '0'),
devtools: process.env.PWDEBUG === 'console',
},
},
},
],

// Folder for test artifacts such as screenshots, videos, traces, etc.
outputDir: 'reports/artifacts',
};

export default config;

Test file (pseudocode)

import { expect, test } from '../fixtures/sometimes-times-out.fixture';

test.describe.configure({ retries: 1 });

it('should check the box using setChecked', async ({ page }) => {
test.skip(true, 'Skipped for some reason');
});

If needed, I can try to create an MRE with a fixture that sometimes times out. I also have the trace of the failure and the HTML report that I can send to a project member privately, but not post to GitHub.

Steps

Run the test until the fixture times out on the first attempt

echo $? to check process return code

Expected

Retried test that skipped during runtime doesn't fail the build

Actual

Retried test that skipped during runtime fails the build