-
Notifications
You must be signed in to change notification settings - Fork 2
NXT-3375: Adapted screenshot tests for chrome 132 #176
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
+252
−40
Merged
Changes from 8 commits
Commits
Show all changes
10 commits
Select commit
Hold shift + click to select a range
b233f31
adapted screenshot tests for chrome 132
daniel-stoian-lgp 40ae51c
reverted page.js
daniel-stoian-lgp 1e23472
removed retries
daniel-stoian-lgp 20be66e
added changelog
daniel-stoian-lgp 4467d63
added retry options for tests
daniel-stoian-lgp 8a42341
added retry options for tests
daniel-stoian-lgp 24af6ee
eslint fix
daniel-stoian-lgp 12950e7
moved session failure check in own functions
daniel-stoian-lgp faa96fe
fixed if else statements
daniel-stoian-lgp d4030cb
eslint fix
daniel-stoian-lgp File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -52,6 +52,9 @@ function initFile (name, content) { | |
| } | ||
|
|
||
| function onPrepare () { | ||
| global.sessionFailures = new Map(); | ||
| global.failedSessions = new Set(); | ||
|
|
||
| if (!fs.existsSync('tests/screenshot/dist/screenshots/reference')) { | ||
| console.log('No reference screenshots found, creating new references!'); | ||
| } | ||
|
|
@@ -62,7 +65,169 @@ function onPrepare () { | |
| return buildApps('screenshot'); | ||
| } | ||
|
|
||
| function beforeTest (testData) { | ||
| /* Checks if a browser session is healthy. If not, it will attempt to recover. */ | ||
| async function checkSessionHealth () { | ||
| const sessionId = browser.sessionId; | ||
|
|
||
| // Check if this session has been marked as dead | ||
| if (global.failedSessions && global.failedSessions.has(sessionId)) { | ||
| throw new Error('Session is marked as failed - skipping remaining tests'); | ||
| } | ||
|
|
||
| // Skip health check if the session was just recovered | ||
| if (global.recentlyRecovered && global.recentlyRecovered.has(sessionId)) { | ||
| global.recentlyRecovered.delete(sessionId); | ||
| } else { | ||
| // Quick health check with a short timeout | ||
| try { | ||
| await Promise.race([ | ||
| browser.execute(() => true), | ||
| new Promise((_, reject) => | ||
| setTimeout(() => reject(new Error('Health check timeout')), 3000) | ||
| ) | ||
| ]); | ||
|
|
||
| // Success - reset failure counter | ||
| if (global.sessionFailures) { | ||
| global.sessionFailures.set(sessionId, 0); | ||
| } | ||
| } catch (e) { | ||
| // Track consecutive failures | ||
| const failures = (global.sessionFailures?.get(sessionId) || 0) + 1; | ||
| if (global.sessionFailures) { | ||
| global.sessionFailures.set(sessionId, failures); | ||
| } | ||
|
|
||
| console.log(`Session ${sessionId} health check failed (failure ${failures}/3)`); | ||
|
|
||
| // Only mark as dead after 3 consecutive failures | ||
| if (failures >= 3) { | ||
| console.log(`Session ${sessionId} has failed 3 times - marking as dead`); | ||
| if (global.failedSessions) { | ||
| global.failedSessions.add(sessionId); | ||
| } | ||
| throw new Error('Session health check failed - marking as dead'); | ||
| } | ||
|
|
||
| // Try quick recovery for the first 2 failures | ||
| console.log(`Attempting quick recovery for session ${sessionId}...`); | ||
| try { | ||
| await browser.reloadSession(); | ||
| await browser.setWindowSize(1920, 1167); | ||
| console.log(`Session ${sessionId} recovered`); | ||
| } catch (recoveryError) { | ||
| console.log(`Recovery attempt failed, will retry next test`); | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| async function cleanUpSessionHealthCheck (testData, error, passed) { | ||
| if (error) { | ||
| const isTimeout = error.message && | ||
| (error.message.includes('timeout') || | ||
| error.message.includes('aborted') || | ||
| error.message.includes('HEADERS_TIMEOUT') || | ||
| error.message.includes('ECONNREFUSED')); | ||
|
|
||
| if (isTimeout) { | ||
| const sessionId = browser.sessionId; | ||
|
|
||
| // Track consecutive failures | ||
| if (!global.sessionFailures) { | ||
| global.sessionFailures = new Map(); | ||
| } | ||
|
|
||
| const failures = (global.sessionFailures.get(sessionId) || 0) + 1; | ||
| global.sessionFailures.set(sessionId, failures); | ||
|
|
||
| console.log(`Timeout #${failures} in session ${sessionId} - test: "${testData.title}"`); | ||
|
|
||
| // Circuit breaker: after 3 consecutive timeouts, kill the session | ||
| if (failures >= 3) { | ||
| console.log(`Session ${sessionId} has failed 3 times consecutively - marking as dead`); | ||
|
|
||
| if (!global.failedSessions) { | ||
| global.failedSessions = new Set(); | ||
| } | ||
| global.failedSessions.add(sessionId); | ||
|
|
||
| // Try to clean up with very short timeout, then give up | ||
| try { | ||
| await Promise.race([ | ||
| (async () => { | ||
| try { | ||
| await browser.execute(() => window.stop()); | ||
| } catch (e) { | ||
| // Ignore | ||
| } | ||
| await browser.deleteSession(); | ||
| })(), | ||
| new Promise((_, reject) => | ||
| setTimeout(() => reject(new Error('Cleanup timeout')), 2000) | ||
| ) | ||
| ]); | ||
| console.log('Session cleanup completed'); | ||
| } catch (e) { | ||
| console.log('Session cleanup timed out - session is dead'); | ||
| } | ||
|
|
||
| return; // Don't try to recover | ||
| } | ||
|
|
||
| // For first 2 failures, attempt recovery | ||
| console.log(`Attempting recovery for session ${sessionId} (attempt ${failures}/3)`); | ||
|
|
||
| try { | ||
| // Try light recovery with timeout | ||
| await Promise.race([ | ||
| (async () => { | ||
| try { | ||
| await browser.execute(() => window.stop()); | ||
| } catch (e) { | ||
| // Ignore | ||
| } | ||
| await browser.deleteSession(); | ||
| await browser.reloadSession(); | ||
| await browser.setWindowSize(1920, 1167); | ||
| await browser.pause(1000); | ||
| })(), | ||
| new Promise((_, reject) => | ||
| setTimeout(() => reject(new Error('Recovery timeout')), 10000) | ||
| ) | ||
| ]); | ||
|
|
||
| console.log('Session recovered successfully'); | ||
|
|
||
| // Reset failure count on successful recovery | ||
| global.sessionFailures.set(sessionId, 0); | ||
|
|
||
| // Mark that we just recovered - skip next health check | ||
| if (!global.recentlyRecovered) { | ||
| global.recentlyRecovered = new Set(); | ||
| } | ||
| global.recentlyRecovered.add(sessionId); | ||
|
|
||
| } catch (recoveryError) { | ||
| console.error(`Session recovery failed: ${recoveryError.message}`); | ||
| } | ||
| } else { | ||
| const sessionId = browser.sessionId; | ||
| if (global.sessionFailures && global.sessionFailures.has(sessionId)) { | ||
| global.sessionFailures.set(sessionId, 0); | ||
| } | ||
| } | ||
| } else if (passed) { | ||
| const sessionId = browser.sessionId; | ||
| if (global.sessionFailures && global.sessionFailures.has(sessionId)) { | ||
| global.sessionFailures.set(sessionId, 0); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| async function beforeTest (testData) { | ||
| await checkSessionHealth(); | ||
|
|
||
| // If title doesn't have a '/', it's not a screenshot test, don't save | ||
| if (testData && testData.title && testData.title.indexOf('/') > 0) { | ||
| const filename = generateReferenceName({test: testData}); | ||
|
|
@@ -78,7 +243,7 @@ function beforeTest (testData) { | |
| } | ||
| } | ||
|
|
||
| function afterTest (testData, _context, {passed}) { | ||
| async function afterTest (testData, _context, {error, passed}) { | ||
| // If this doesn't include context data, not a screenshot test | ||
| if (testData && testData.title && testData.context && testData.context.params) { | ||
| const fileName = testData.context.fileName.replace(/ /g, '_') + '.png'; | ||
|
|
@@ -99,22 +264,42 @@ function afterTest (testData, _context, {passed}) { | |
| } | ||
|
|
||
| if (!passed) { | ||
| const screenPath = path.join(screenshotRelativePath, 'actual', fileName); | ||
| const diffPath = path.join(screenshotRelativePath, 'diff', fileName); | ||
| fs.open(failedScreenshotFilename, 'a', (err, fd) => { | ||
| if (err) { | ||
| console.error('Unable to create failed test log file!'); | ||
| } else { | ||
| const title = testData.title.replace(/~\//g, '/'); | ||
| const {params, url} = testData.context; | ||
| const output = {title, diffPath, referencePath, screenPath, params, url}; | ||
| fs.appendFile(fd, `${JSON.stringify(output)},`, 'utf8', () => { | ||
| fs.close(fd); | ||
| }); | ||
| } | ||
| }); | ||
| // Track failed tests to avoid duplicate logging during retries | ||
| if (!global.loggedFailures) { | ||
| global.loggedFailures = new Set(); | ||
| } | ||
|
|
||
| const testIdentifier = testData.title + '::' + fileName; | ||
|
|
||
| // Only log if we haven't already logged this test failure | ||
| if (!global.loggedFailures.has(testIdentifier)) { | ||
| global.loggedFailures.add(testIdentifier); | ||
|
|
||
| const screenPath = path.join(screenshotRelativePath, 'actual', fileName); | ||
| const diffPath = path.join(screenshotRelativePath, 'diff', fileName); | ||
| fs.open(failedScreenshotFilename, 'a', (err, fd) => { | ||
| if (err) { | ||
| console.error('Unable to create failed test log file!'); | ||
| } else { | ||
| const title = testData.title.replace(/~\//g, '/'); | ||
| const {params, url} = testData.context; | ||
| const output = {title, diffPath, referencePath, screenPath, params, url}; | ||
| fs.appendFile(fd, `${JSON.stringify(output)},`, 'utf8', () => { | ||
| fs.close(fd); | ||
| }); | ||
| } | ||
| }); | ||
| } | ||
| } else if (passed) { | ||
|
||
| // Test passed on retry - remove from logged failures | ||
| if (global.loggedFailures) { | ||
| const testIdentifier = testData.title + '::' + fileName; | ||
| global.loggedFailures.delete(testIdentifier); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| await cleanUpSessionHealthCheck(testData, error, passed); | ||
| } | ||
|
|
||
| function onComplete () { | ||
|
|
||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is this comment still needed?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
at this moment, yes. We have to decide how we set up the chrome version from Jenkins. I don't think the current ui-test-utils configuration works for multiple chrome versions