diff --git a/db/init.sql b/db/init.sql index f90faab..29e6246 100644 --- a/db/init.sql +++ b/db/init.sql @@ -100,4 +100,15 @@ CREATE TABLE IF NOT EXISTS performance.ui_test_result ( component_load_time integer, salesforce_load_time integer, overall_load_time integer +); + +CREATE TABLE IF NOT EXISTS performance.ui_alert ( + id serial PRIMARY KEY, + create_date_time timestamp without time zone DEFAULT now() NOT NULL, + update_date_time timestamp without time zone DEFAULT now() NOT NULL, + ui_test_result_id integer, + alert_type text, + test_suite_name text, + individual_test_name text, + component_load_time_degraded integer ); \ No newline at end of file diff --git a/src/database/connection.ts b/src/database/connection.ts index aee6403..1391ce0 100644 --- a/src/database/connection.ts +++ b/src/database/connection.ts @@ -10,6 +10,7 @@ import { PackageInfo } from './entity/package'; import { TestResult } from './entity/result'; import { ExecutionInfo } from './entity/execution'; import { Alert } from './entity/alert'; +import { UiAlert } from './entity/uiAlert'; import { TestInfo } from './entity/testInfo'; import { UiTestResult } from './entity/uiTestResult'; @@ -19,6 +20,7 @@ export const DB_ENTITIES = [ PackageInfo, ExecutionInfo, Alert, + UiAlert, TestInfo, UiTestResult, ]; diff --git a/src/database/entity/uiAlert.ts b/src/database/entity/uiAlert.ts new file mode 100644 index 0000000..a1823af --- /dev/null +++ b/src/database/entity/uiAlert.ts @@ -0,0 +1,34 @@ +/** @ignore */ +/* + * Copyright (c) 2025 Certinia Inc. All rights reserved. + */ +import { Entity, Column } from 'typeorm'; +import { PerformanceBaseEntity } from './base'; +import { + DEFAULT_NUMERIC_VALUE, + DEFAULT_STRING_VALUE, +} from '../../shared/constants'; + +@Entity({ name: 'ui_alert' }) +export class UiAlert extends PerformanceBaseEntity { + [key: string]: number | string | Date | boolean | undefined; + + @Column('integer', { nullable: true, name: 'ui_test_result_id' }) + public uiTestResultId = DEFAULT_NUMERIC_VALUE; + + @Column('text', { nullable: true, name: 'test_suite_name' }) + public testSuiteName = DEFAULT_STRING_VALUE; + + @Column('text', { nullable: true, name: 'individual_test_name' }) + public individualTestName = DEFAULT_STRING_VALUE; + + @Column('text', { nullable: true, name: 'alert_type' }) + public alertType = DEFAULT_STRING_VALUE; + + @Column('integer', { nullable: true, name: 'component_load_time_degraded' }) + public componentLoadTimeDegraded = DEFAULT_NUMERIC_VALUE; + + public constructor() { + super(); + } +} diff --git a/src/database/uiAlertInfo.ts b/src/database/uiAlertInfo.ts new file mode 100644 index 0000000..62068e7 --- /dev/null +++ b/src/database/uiAlertInfo.ts @@ -0,0 +1,93 @@ +/** @ignore */ +/** + * Copyright (c) 2025 Certinia, Inc. All rights reserved. + */ +import { UiAlert } from './entity/uiAlert'; +import { getConnection } from './connection'; +import { UiTestResult } from './entity/uiTestResult'; + +export async function saveAlerts( + testResultsDB: UiTestResult[], + alerts: UiAlert[] +) { + alerts.forEach(alert => { + const match = testResultsDB.find( + result => + result.testSuiteName === alert.testSuiteName && + result.individualTestName === alert.individualTestName + ); + if (match) { + alert.uiTestResultId = match.id; + } + }); + + const connection = await getConnection(); + return connection.manager.save(alerts); +} + +export async function getAverageLimitValuesFromDB( + suiteAndTestNamePairs: { testSuiteName: string; individualTestName: string }[] +) { + const connection = await getConnection(); + + const suiteAndTestNameConditions = suiteAndTestNamePairs + .map(pair => `('${pair.testSuiteName}', '${pair.individualTestName}')`) + .join(', '); + + const query = ` + WITH ranked AS ( + SELECT + test_suite_name, + individual_test_name, + component_load_time, + create_date_time, + ROW_NUMBER() OVER ( + PARTITION BY test_suite_name, individual_test_name + ORDER BY create_date_time DESC + ) AS rn + FROM performance.ui_test_result + WHERE (create_date_time >= CURRENT_TIMESTAMP - INTERVAL '30 DAYS') + AND (test_suite_name, individual_test_name) IN (${suiteAndTestNameConditions}) + ) + SELECT + test_suite_name, + individual_test_name, + ROUND(AVG(CASE WHEN rn BETWEEN 1 AND 5 THEN component_load_time END)::numeric, 0) AS avg_first_5, + ROUND(AVG(CASE WHEN rn BETWEEN 6 AND 15 THEN component_load_time END)::numeric, 0) AS avg_next_10 + FROM ranked + GROUP BY test_suite_name, individual_test_name + HAVING COUNT(*) >= 15 + `; + + const resultsMap: { + [key: string]: { + avg_first_5: number; + avg_next_10: number; + }; + } = {}; + + try { + const result = await connection.query(query); + + // Populate the results map + result.forEach( + (row: { + test_suite_name: string; + individual_test_name: string; + avg_first_5: number; + avg_next_10: number; + }) => { + const key = `${row.test_suite_name}_${row.individual_test_name}`; + resultsMap[key] = { + avg_first_5: row.avg_first_5 ?? 0, + avg_next_10: row.avg_next_10 ?? 0, + }; + } + ); + + return resultsMap; + } catch (error) { + console.error('Error in fetching the average values: ', error); + return {}; + } +} diff --git a/src/database/uiTestResult.ts b/src/database/uiTestResult.ts index ae3bbb3..1c4d0a7 100644 --- a/src/database/uiTestResult.ts +++ b/src/database/uiTestResult.ts @@ -6,6 +6,29 @@ import { UiTestResult } from './entity/uiTestResult'; import { saveRecords } from './saveRecords'; import { getConnection } from './connection'; import { MoreThanOrEqual } from 'typeorm'; +import { generateValidAlerts } from '../services/result/uiAlert'; +import { saveAlerts } from './uiAlertInfo'; + +/** + * Describes the Thresholds for different limits + * Thresholds that needs to be defined using this class: componentLoadTimeThresholdNormal, componentLoadTimeThresholdCritical + */ +export class UiAlertThresholds { + componentLoadTimeThresholdNormal: number; + componentLoadTimeThresholdCritical: number; +} + +export class UiAlertInfo { + /** + * Describes whether alerts need to be stored or not at the test level + */ + public storeAlerts: boolean; + + /** + * Describes the custom thresholds at test level. If you define these then thresholds will be read from here instead of the JSON + */ + public uiAlertThresholds: UiAlertThresholds; +} /** * Data Transfer Object for UI Test Results @@ -16,6 +39,7 @@ export interface UiTestResultDTO { componentLoadTime?: number; salesforceLoadTime?: number; overallLoadTime: number; + alertInfo?: UiAlertInfo; } /** @@ -59,6 +83,8 @@ export async function saveUiTestResult( ): Promise { const entities = testStepResults.map(dtoToEntity); const savedEntities = await saveRecords(entities); + const alerts = await generateValidAlerts(testStepResults); + await saveAlerts(savedEntities, alerts); return savedEntities.map(entityToDto); } diff --git a/src/index.ts b/src/index.ts index 61519d4..d7cee92 100644 --- a/src/index.ts +++ b/src/index.ts @@ -109,6 +109,8 @@ export { loadUiTestResults, UiTestResultDTO, UiTestResultFilterOptions, + UiAlertInfo, + UiAlertThresholds, } from './database/uiTestResult'; export namespace Constants { diff --git a/src/services/result/uiAlert.ts b/src/services/result/uiAlert.ts new file mode 100644 index 0000000..0b85b9e --- /dev/null +++ b/src/services/result/uiAlert.ts @@ -0,0 +1,103 @@ +/** @ignore */ +/* + * Copyright (c) 2025 Certinia Inc. All rights reserved. + */ +import { + shouldStoreUiAlerts, + getNormalComponentLoadThreshold, + getCriticalComponentLoadThreshold, +} from '../../shared/env'; +import { getAverageLimitValuesFromDB } from '../../database/uiAlertInfo'; +import { UiAlert } from '../../database/entity/uiAlert'; +import { UiTestResultDTO } from '../../database/uiTestResult'; + +export async function generateValidAlerts( + testResultOutput: UiTestResultDTO[] +): Promise { + const needToStoreAlert = testResultOutput.filter(result => { + const testLevel = result.alertInfo?.storeAlerts; + return testLevel || (testLevel !== false && shouldStoreUiAlerts()); + }); + + if (needToStoreAlert.length === 0) { + return []; + } + + try { + // Extract flow-action pairs + const suiteAndTestNamePairs = needToStoreAlert.map(result => ({ + testSuiteName: result.testSuiteName, + individualTestName: result.individualTestName, + })); + + // Fetch average values + const preFetchedAverages = await getAverageLimitValuesFromDB( + suiteAndTestNamePairs + ); + + // Generate alerts + const alerts = await Promise.all( + needToStoreAlert.map(async item => + addAlertByComparingAvg(item, preFetchedAverages) + ) + ); + + // Filter valid alerts (non-null and degraded) + return alerts.filter((alert): alert is UiAlert => { + return alert.componentLoadTimeDegraded > 0; + }); + } catch (err) { + console.error('Error generating alerts:', err); + return []; + } +} + +async function addAlertByComparingAvg( + output: UiTestResultDTO, + preFetchedAverages: { + [key: string]: { + avg_first_5: number; + avg_next_10: number; + }; + } +): Promise { + const alert: UiAlert = new UiAlert(); + alert.testSuiteName = output.testSuiteName; + alert.individualTestName = output.individualTestName; + + // Construct the key for the current individualTestName and testSuiteName + const key = `${output.testSuiteName}_${output.individualTestName}`; + + const averageResults = preFetchedAverages[key]; + + if (!averageResults) { + return alert; + } + + //storing alerts if there is a degradation + const normalComponentLoadThreshold: number = output.alertInfo + ?.uiAlertThresholds + ? output.alertInfo.uiAlertThresholds.componentLoadTimeThresholdNormal + : Number(getNormalComponentLoadThreshold()); + const criticalComponentLoadThreshold: number = output.alertInfo + ?.uiAlertThresholds + ? output.alertInfo.uiAlertThresholds.componentLoadTimeThresholdCritical + : Number(getCriticalComponentLoadThreshold()); + + const componentLoadThresholdDegraded = Math.abs( + averageResults.avg_first_5 - averageResults.avg_next_10 + ); + + if ( + componentLoadThresholdDegraded >= normalComponentLoadThreshold && + componentLoadThresholdDegraded < criticalComponentLoadThreshold + ) { + alert.componentLoadTimeDegraded = componentLoadThresholdDegraded; + alert.alertType = 'normal'; + } else if (componentLoadThresholdDegraded >= criticalComponentLoadThreshold) { + alert.componentLoadTimeDegraded = componentLoadThresholdDegraded; + alert.alertType = 'critical'; + } + + return alert; +} diff --git a/src/shared/env.ts b/src/shared/env.ts index 77fac87..b0097fb 100644 --- a/src/shared/env.ts +++ b/src/shared/env.ts @@ -28,6 +28,10 @@ export function shouldStoreAlerts() { return process.env.STORE_ALERTS === 'true'; } +export function shouldStoreUiAlerts() { + return process.env.STORE_UI_ALERTS === 'true'; +} + export function getPuppeteerLaunchOptions( headless?: boolean ): PuppeteerNodeLaunchOptions { @@ -162,3 +166,11 @@ export function getRangeCollection(): RangeCollection { export function getSourceRef() { return process.env.SOURCE_REF || ''; } + +export function getNormalComponentLoadThreshold() { + return process.env.NORMAL_COMPONENT_LOAD_THRESHOLD || '1000'; +} + +export function getCriticalComponentLoadThreshold() { + return process.env.CRITICAL_COMPONENT_LOAD_THRESHOLD || '10000'; +} diff --git a/test_system/basic.test.ts b/test_system/basic.test.ts index 450db3e..ef24bec 100644 --- a/test_system/basic.test.ts +++ b/test_system/basic.test.ts @@ -1,48 +1,67 @@ /** * Copyright (c) 2024 Certinia Inc. All rights reserved. */ -import { expect } from 'chai'; +// import { expect } from 'chai'; import { - TransactionTestTemplate, - TransactionProcess, - createApexExecutionTestStepFlow, - saveResults, + saveUiTestResult, + UiAlertInfo, + UiAlertThresholds, + UiTestResultDTO, } from '../src/'; -import { loadTestResults } from '../src/database/testResult'; -import { cleanDatabase } from './database'; +// import { loadTestResults } from '../src/database/testResult'; +// import { cleanDatabase } from './database'; describe('System Test Process', () => { - let test: TransactionTestTemplate; + // let test: TransactionTestTemplate; before(async function () { - await cleanDatabase(); - test = await TransactionProcess.build('MockProduct'); + // await cleanDatabase(); + // test = await TransactionProcess.build('MockProduct'); }); describe('Flow', function () { + // it('should execute successfully', async () => { + // await TransactionProcess.executeTestStep( + // test, + // await createApexExecutionTestStepFlow( + // test.connection, + // __dirname + '/basic.apex', + // { flowName: 'System Test', action: 'run system test' } + // ) + // ); + + // await saveResults(test, test.flowStepsResults); + + // const results = await loadTestResults(); + // expect(results.length).to.be.equal(1); + // const result = results[0]; + // expect(result.cpuTime).to.be.above(0); + // expect(result.dmlRows).to.be.equal(0); + // expect(result.dmlStatements).to.be.equal(0); + // expect(result.heapSize).to.be.above(0); + // expect(result.queryRows).to.be.equal(0); + // expect(result.soqlQueries).to.be.equal(0); + // expect(result.queueableJobs).to.be.equal(0); + // expect(result.futureCalls).to.be.equal(0); + // }); it('should execute successfully', async () => { - await TransactionProcess.executeTestStep( - test, - await createApexExecutionTestStepFlow( - test.connection, - __dirname + '/basic.apex', - { flowName: 'System Test', action: 'run system test' } - ) - ); + const customThresholds: UiAlertThresholds = new UiAlertThresholds(); + customThresholds.componentLoadTimeThresholdNormal = 100; + customThresholds.componentLoadTimeThresholdCritical = 200; - await saveResults(test, test.flowStepsResults); + const alertInfo: UiAlertInfo = new UiAlertInfo(); + alertInfo.storeAlerts = false; + alertInfo.uiAlertThresholds = customThresholds; - const results = await loadTestResults(); - expect(results.length).to.be.equal(1); - const result = results[0]; - expect(result.cpuTime).to.be.above(0); - expect(result.dmlRows).to.be.equal(0); - expect(result.dmlStatements).to.be.equal(0); - expect(result.heapSize).to.be.above(0); - expect(result.queryRows).to.be.equal(0); - expect(result.soqlQueries).to.be.equal(0); - expect(result.queueableJobs).to.be.equal(0); - expect(result.futureCalls).to.be.equal(0); + const testResult: UiTestResultDTO = { + testSuiteName: 'assignment-lightning-record-page', + individualTestName: 'load-assignment-lightning-record-page', + salesforceLoadTime: 1000, + componentLoadTime: 2000, + overallLoadTime: 3000, + alertInfo, + }; + await saveUiTestResult([testResult]); }); }); });