diff --git a/CHANGELOG.md b/CHANGELOG.md index 17dcb2d..c05ed47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Benchmarker - Changelog +## 6.0.0 + +### Breaking Changes + +- Database schema updated: Use `db/migrations/V6__create_performance_ui_alert_table.sql` script to create new table to store alerts for performance UI test results. + ## 5.0.0 ### Breaking Changes diff --git a/db/init.sql b/db/init.sql index f90faab..29e6246 100644 --- a/db/init.sql +++ b/db/init.sql @@ -100,4 +100,15 @@ CREATE TABLE IF NOT EXISTS performance.ui_test_result ( component_load_time integer, salesforce_load_time integer, overall_load_time integer +); + +CREATE TABLE IF NOT EXISTS performance.ui_alert ( + id serial PRIMARY KEY, + create_date_time timestamp without time zone DEFAULT now() NOT NULL, + update_date_time timestamp without time zone DEFAULT now() NOT NULL, + ui_test_result_id integer, + alert_type text, + test_suite_name text, + individual_test_name text, + component_load_time_degraded integer ); \ No newline at end of file diff --git a/db/migrations/V6__create_performance_ui_alert_table.sql b/db/migrations/V6__create_performance_ui_alert_table.sql new file mode 100644 index 0000000..89786bd --- /dev/null +++ b/db/migrations/V6__create_performance_ui_alert_table.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS performance.ui_alert ( + id serial PRIMARY KEY, + create_date_time timestamp without time zone DEFAULT now() NOT NULL, + update_date_time timestamp without time zone DEFAULT now() NOT NULL, + ui_test_result_id integer, + alert_type text, + test_suite_name text, + individual_test_name text, + component_load_time_degraded integer +); \ No newline at end of file diff --git a/docs/user/alerts.md b/docs/user/alerts.md index 8441293..3fd9c92 100644 --- a/docs/user/alerts.md +++ b/docs/user/alerts.md @@ -110,3 +110,48 @@ await TransactionProcess.executeTestStep(..., alertInfo); ```txt Note: If the test level threshold is misconfigured below the average, you get an alert with a value of 0. Recommend filtering out zero alerts when querying for new records. ``` +# UI Alerts + +UI Alerts can be stored in the database to monitor performance degradation over time. Each record’s degradation value represents the difference between the average of the first 5 ui test results and the average of the subsequent 10 ui test results recorded within the past 30 days. A minimum of 15 results within a 30-day window is required to trigger UI alert storage. + +## Usage + +### Environment Variables + +To enable UI alerts globally, set the `STORE_UI_ALERTS` variable to `true` in the environment file + +To override the default normal component load time threshold (1000 ms), set the `NORMAL_COMPONENT_LOAD_THRESHOLD` variable to any desired value in the environment file + +To override the default critical component load time threshold (10000 ms), set the `CRITICAL_COMPONENT_LOAD_THRESHOLD` variable to any desired value in the environment file + +#### Example Scenario + +* The average component load time for the first 5 results was 1500 ms, while the next 10 results averaged 1000 ms. +* You will get an normal alert, saying it has degraded by 500 above the average. + +### Test Level Thresholds + +Alternatively, thresholds can be configured at the test level. If the difference between the average of the first 5 and the subsequent 10 ui test results exceeds the defined custom threshold, an alert will be stored. The degradation value remains the difference between the average of the first 5 results and the average of the next 10 results recorded. The threshold simply determines whether an alert is triggered. + +```ts +// Replace global alert behaviour with exact thresholds +const customThresholds: UiAlertThresholds = new UiAlertThresholds(); +customThresholds.componentLoadTimeThresholdNormal = 50; +customThresholds.componentLoadTimeThresholdCritical = 100; + +// Enable alerting for this test if not already active +const alertInfo: UiAlertInfo = new UiAlertInfo(); +alertInfo.storeAlerts = true; +alertInfo.uiAlertThresholds = customThresholds; + +const testResult: UiTestResultDTO = { + ..., + alertInfo, + }; + +await saveUiTestResult([testResult]); +``` +#### Example Scenario + +* Test-level thresholds for normal and critical component load times are set to 50 ms and 100 ms, respectively. The average load time for the first 5 results was 1500 ms, while the subsequent 10 results averaged 1000 ms. +* You will get an critical alert, saying it has degraded by 500 above the average. diff --git a/src/database/connection.ts b/src/database/connection.ts index aee6403..76c71b7 100644 --- a/src/database/connection.ts +++ b/src/database/connection.ts @@ -12,6 +12,7 @@ import { ExecutionInfo } from './entity/execution'; import { Alert } from './entity/alert'; import { TestInfo } from './entity/testInfo'; import { UiTestResult } from './entity/uiTestResult'; +import { UiAlert } from './entity/uiAlert'; export const DB_ENTITIES = [ TestResult, @@ -21,6 +22,7 @@ export const DB_ENTITIES = [ Alert, TestInfo, UiTestResult, + UiAlert, ]; let connection: DataSource | null = null; diff --git a/src/database/entity/uiAlert.ts b/src/database/entity/uiAlert.ts new file mode 100644 index 0000000..a1823af --- /dev/null +++ b/src/database/entity/uiAlert.ts @@ -0,0 +1,34 @@ +/** @ignore */ +/* + * Copyright (c) 2025 Certinia Inc. All rights reserved. + */ +import { Entity, Column } from 'typeorm'; +import { PerformanceBaseEntity } from './base'; +import { + DEFAULT_NUMERIC_VALUE, + DEFAULT_STRING_VALUE, +} from '../../shared/constants'; + +@Entity({ name: 'ui_alert' }) +export class UiAlert extends PerformanceBaseEntity { + [key: string]: number | string | Date | boolean | undefined; + + @Column('integer', { nullable: true, name: 'ui_test_result_id' }) + public uiTestResultId = DEFAULT_NUMERIC_VALUE; + + @Column('text', { nullable: true, name: 'test_suite_name' }) + public testSuiteName = DEFAULT_STRING_VALUE; + + @Column('text', { nullable: true, name: 'individual_test_name' }) + public individualTestName = DEFAULT_STRING_VALUE; + + @Column('text', { nullable: true, name: 'alert_type' }) + public alertType = DEFAULT_STRING_VALUE; + + @Column('integer', { nullable: true, name: 'component_load_time_degraded' }) + public componentLoadTimeDegraded = DEFAULT_NUMERIC_VALUE; + + public constructor() { + super(); + } +} diff --git a/src/database/uiAlertInfo.ts b/src/database/uiAlertInfo.ts new file mode 100644 index 0000000..6522308 --- /dev/null +++ b/src/database/uiAlertInfo.ts @@ -0,0 +1,93 @@ +/** @ignore */ +/** + * Copyright (c) 2025 Certinia, Inc. All rights reserved. + */ +import { getConnection } from './connection'; +import { UiAlert } from './entity/uiAlert'; +import { UiTestResult } from './entity/uiTestResult'; + +export async function saveAlerts( + testResultsDB: UiTestResult[], + alerts: UiAlert[] +) { + alerts.forEach(alert => { + const match = testResultsDB.find( + result => + result.testSuiteName === alert.testSuiteName && + result.individualTestName === alert.individualTestName + ); + if (match) { + alert.uiTestResultId = match.id; + } + }); + + const connection = await getConnection(); + return connection.manager.save(alerts); +} + +export async function getAverageLimitValuesFromDB( + suiteAndTestNamePairs: { testSuiteName: string; individualTestName: string }[] +) { + const connection = await getConnection(); + + const suiteAndTestNameConditions = suiteAndTestNamePairs + .map(pair => `('${pair.testSuiteName}', '${pair.individualTestName}')`) + .join(', '); + + const query = ` + WITH ranked AS ( + SELECT + test_suite_name, + individual_test_name, + component_load_time, + create_date_time, + ROW_NUMBER() OVER ( + PARTITION BY test_suite_name, individual_test_name + ORDER BY create_date_time DESC + ) AS rn + FROM performance.ui_test_result + WHERE (create_date_time >= CURRENT_TIMESTAMP - INTERVAL '30 DAYS') + AND (test_suite_name, individual_test_name) IN (${suiteAndTestNameConditions}) + ) + SELECT + test_suite_name, + individual_test_name, + ROUND(AVG(CASE WHEN rn BETWEEN 1 AND 5 THEN component_load_time END)::numeric, 0) AS avg_first_5, + ROUND(AVG(CASE WHEN rn BETWEEN 6 AND 15 THEN component_load_time END)::numeric, 0) AS avg_next_10 + FROM ranked + GROUP BY test_suite_name, individual_test_name + HAVING COUNT(*) >= 15 + `; + + const resultsMap: { + [key: string]: { + avg_first_5: number; + avg_next_10: number; + }; + } = {}; + + try { + const result = await connection.query(query); + + // Populate the results map + result.forEach( + (row: { + test_suite_name: string; + individual_test_name: string; + avg_first_5: number; + avg_next_10: number; + }) => { + const key = `${row.test_suite_name}_${row.individual_test_name}`; + resultsMap[key] = { + avg_first_5: row.avg_first_5 ?? 0, + avg_next_10: row.avg_next_10 ?? 0, + }; + } + ); + + return resultsMap; + } catch (error) { + console.error('Error in fetching the average values: ', error); + return {}; + } +} diff --git a/src/database/uiTestResult.ts b/src/database/uiTestResult.ts index ae3bbb3..b9c99e8 100644 --- a/src/database/uiTestResult.ts +++ b/src/database/uiTestResult.ts @@ -6,6 +6,29 @@ import { UiTestResult } from './entity/uiTestResult'; import { saveRecords } from './saveRecords'; import { getConnection } from './connection'; import { MoreThanOrEqual } from 'typeorm'; +import { generateValidAlerts } from '../services/result/uiAlert'; +import { saveAlerts } from './uiAlertInfo'; + +/** + * Describes the Thresholds for different limits + * Use this class to specify thresholds such as: componentLoadTimeThresholdNormal and componentLoadTimeThresholdCritical. + */ +export class UiAlertThresholds { + componentLoadTimeThresholdNormal: number; + componentLoadTimeThresholdCritical: number; +} + +export class UiAlertInfo { + /** + * Describes whether alerts need to be stored or not at the test level + */ + public storeAlerts: boolean; + + /** + * Defines custom thresholds at the test level. When specified, these values take precedence over defaults and those set in the environment file. + */ + public uiAlertThresholds: UiAlertThresholds; +} /** * Data Transfer Object for UI Test Results @@ -16,6 +39,7 @@ export interface UiTestResultDTO { componentLoadTime?: number; salesforceLoadTime?: number; overallLoadTime: number; + alertInfo?: UiAlertInfo; } /** @@ -59,6 +83,10 @@ export async function saveUiTestResult( ): Promise { const entities = testStepResults.map(dtoToEntity); const savedEntities = await saveRecords(entities); + const alerts = await generateValidAlerts(testStepResults); + if (alerts.length) { + await saveAlerts(savedEntities, alerts); + } return savedEntities.map(entityToDto); } diff --git a/src/index.ts b/src/index.ts index 61519d4..d7cee92 100644 --- a/src/index.ts +++ b/src/index.ts @@ -109,6 +109,8 @@ export { loadUiTestResults, UiTestResultDTO, UiTestResultFilterOptions, + UiAlertInfo, + UiAlertThresholds, } from './database/uiTestResult'; export namespace Constants { diff --git a/src/services/defaultRanges.ts b/src/services/defaultRanges.ts index 5804c43..fcb5006 100644 --- a/src/services/defaultRanges.ts +++ b/src/services/defaultRanges.ts @@ -6,16 +6,14 @@ import { RangeCollection } from './ranges'; export const DEFAULT_RANGES: RangeCollection = { dml_ranges: [ - { start_range: 0, end_range: 50, offset_threshold: 5 }, - { start_range: 51, end_range: 100, offset_threshold: 3 }, - { start_range: 101, end_range: 120, offset_threshold: 2 }, - { start_range: 121, end_range: 149, offset_threshold: 1 }, + { start_range: 0, end_range: 50, offset_threshold: 3 }, + { start_range: 51, end_range: 100, offset_threshold: 2 }, + { start_range: 101, end_range: 149, offset_threshold: 1 }, ], soql_ranges: [ - { start_range: 0, end_range: 50, offset_threshold: 10 }, - { start_range: 51, end_range: 80, offset_threshold: 5 }, - { start_range: 81, end_range: 90, offset_threshold: 2 }, - { start_range: 91, end_range: 99, offset_threshold: 1 }, + { start_range: 0, end_range: 30, offset_threshold: 3 }, + { start_range: 31, end_range: 60, offset_threshold: 2 }, + { start_range: 61, end_range: 99, offset_threshold: 1 }, ], cpu_ranges: [ { start_range: 0, end_range: 2000, offset_threshold: 3000 }, diff --git a/src/services/result/uiAlert.ts b/src/services/result/uiAlert.ts new file mode 100644 index 0000000..8dc001a --- /dev/null +++ b/src/services/result/uiAlert.ts @@ -0,0 +1,103 @@ +/** @ignore */ +/* + * Copyright (c) 2025 Certinia Inc. All rights reserved. + */ +import { + shouldStoreUiAlerts, + getNormalComponentLoadThreshold, + getCriticalComponentLoadThreshold, +} from '../../shared/env'; +import { getAverageLimitValuesFromDB } from '../../database/uiAlertInfo'; +import { UiAlert } from '../../database/entity/uiAlert'; +import { UiTestResultDTO } from '../../database/uiTestResult'; +import { NORMAL, CRITICAL } from '../../shared/constants'; + +export async function generateValidAlerts( + testResultOutput: UiTestResultDTO[] +): Promise { + const needToStoreAlert = testResultOutput.filter(result => { + const testLevel = result.alertInfo?.storeAlerts; + return testLevel || (testLevel !== false && shouldStoreUiAlerts()); + }); + + if (needToStoreAlert.length === 0) { + return []; + } + + try { + // Extract flow-action pairs + const suiteAndTestNamePairs = needToStoreAlert.map(result => ({ + testSuiteName: result.testSuiteName, + individualTestName: result.individualTestName, + })); + + // Fetch average values + const preFetchedAverages = await getAverageLimitValuesFromDB( + suiteAndTestNamePairs + ); + + // Generate alerts + const alerts = await Promise.all( + needToStoreAlert.map(async item => + addAlertByComparingAvg(item, preFetchedAverages) + ) + ); + + // Filter valid alerts (non-null and degraded) + return alerts.filter((alert): alert is UiAlert => { + return alert.componentLoadTimeDegraded > 0; + }); + } catch (err) { + console.error('Error generating alerts:', err); + return []; + } +} + +async function addAlertByComparingAvg( + output: UiTestResultDTO, + preFetchedAverages: { + [key: string]: { + avg_first_5: number; + avg_next_10: number; + }; + } +): Promise { + const alert: UiAlert = new UiAlert(); + alert.testSuiteName = output.testSuiteName; + alert.individualTestName = output.individualTestName; + + // Construct the key for the current individualTestName and testSuiteName + const key = `${output.testSuiteName}_${output.individualTestName}`; + + const averageResults = preFetchedAverages[key]; + + if (!averageResults) { + return alert; + } + + // Storing alerts if there is a degradation + const normalComponentLoadThreshold: number = output.alertInfo + ?.uiAlertThresholds + ? output.alertInfo.uiAlertThresholds.componentLoadTimeThresholdNormal + : Number(getNormalComponentLoadThreshold()); + const criticalComponentLoadThreshold: number = output.alertInfo + ?.uiAlertThresholds + ? output.alertInfo.uiAlertThresholds.componentLoadTimeThresholdCritical + : Number(getCriticalComponentLoadThreshold()); + const componentLoadThresholdDegraded = Math.abs( + averageResults.avg_first_5 - averageResults.avg_next_10 + ); + + if ( + componentLoadThresholdDegraded >= normalComponentLoadThreshold && + componentLoadThresholdDegraded < criticalComponentLoadThreshold + ) { + alert.componentLoadTimeDegraded = componentLoadThresholdDegraded; + alert.alertType = NORMAL; + } else if (componentLoadThresholdDegraded >= criticalComponentLoadThreshold) { + alert.componentLoadTimeDegraded = componentLoadThresholdDegraded; + alert.alertType = CRITICAL; + } + + return alert; +} diff --git a/src/shared/constants.ts b/src/shared/constants.ts index 62b8aec..0126501 100644 --- a/src/shared/constants.ts +++ b/src/shared/constants.ts @@ -59,3 +59,8 @@ export const CSV_EID = ''; ******************************************************************************/ export const ERROR_OPENING_BROWSER = 'No test will be executed, a problem was found during browser opening and login'; +/***************************************************************************** + * UI ALERT TYPES + ******************************************************************************/ +export const NORMAL = 'normal'; +export const CRITICAL = 'critical'; diff --git a/src/shared/env.ts b/src/shared/env.ts index 77fac87..a2824a7 100644 --- a/src/shared/env.ts +++ b/src/shared/env.ts @@ -28,6 +28,10 @@ export function shouldStoreAlerts() { return process.env.STORE_ALERTS === 'true'; } +export function shouldStoreUiAlerts() { + return process.env.STORE_UI_ALERTS === 'true'; +} + export function getPuppeteerLaunchOptions( headless?: boolean ): PuppeteerNodeLaunchOptions { @@ -162,3 +166,11 @@ export function getRangeCollection(): RangeCollection { export function getSourceRef() { return process.env.SOURCE_REF || ''; } + +export function getNormalComponentLoadThreshold() { + return process.env.NORMAL_COMPONENT_LOAD_THRESHOLD || '1000'; +} + +export function getCriticalComponentLoadThreshold() { + return process.env.CRITICAL_COMPONENT_LOAD_THRESHOLD || '10000'; +} \ No newline at end of file diff --git a/test/database/uiAlertInfo.test.ts b/test/database/uiAlertInfo.test.ts new file mode 100644 index 0000000..32f7ebe --- /dev/null +++ b/test/database/uiAlertInfo.test.ts @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2025 Certinia, inc. All rights reserved. + */ + +import { + getAverageLimitValuesFromDB, + saveAlerts, +} from '../../src/database/uiAlertInfo'; +import * as db from '../../src/database/connection'; +import sinon from 'sinon'; +import chai, { expect } from 'chai'; +import sinonChai from 'sinon-chai'; +import { DataSource } from 'typeorm'; +import { UiAlert } from '../../src/database/entity/uiAlert'; +import { UiTestResult } from '../../src/database/entity/uiTestResult'; + +chai.use(sinonChai); + +describe('src/database/alertInfo', () => { + let mockQuery: sinon.SinonStub; + let connectionStub: sinon.SinonStub; + let mockDataSource: any; + + beforeEach(() => { + mockQuery = sinon.stub(); + mockDataSource = { query: mockQuery }; + connectionStub = sinon.stub(db, 'getConnection').resolves(mockDataSource); + }); + + afterEach(() => { + sinon.restore(); + }); + + describe('getAverageLimitValuesFromDB', () => { + it('should return average limit values for valid data', async () => { + // Given + const suiteAndTestNamePairs = [ + { + testSuiteName: 'testSuiteName1', + individualTestName: 'individualTestName1', + }, + { + testSuiteName: 'testSuiteName2', + individualTestName: 'individualTestName2', + }, + ]; + + const mockResults = [ + { + test_suite_name: 'testSuiteName1', + individual_test_name: 'individualTestName1', + avg_first_5: 2000, + avg_next_10: 1500, + }, + { + test_suite_name: 'testSuiteName2', + individual_test_name: 'individualTestName2', + avg_first_5: 2000, + avg_next_10: 1500, + }, + ]; + + mockQuery.resolves(mockResults); + + // When + const results = await getAverageLimitValuesFromDB(suiteAndTestNamePairs); + + // Then + expect(mockQuery.calledOnce).to.be.true; + expect(mockQuery.args[0][0]).to.include('SELECT'); + expect(mockQuery.args[0][0]).to.include( + "(test_suite_name, individual_test_name) IN (('testSuiteName1', 'individualTestName1'), ('testSuiteName2', 'individualTestName2'))" + ); + + expect(results).to.deep.equal({ + testSuiteName1_individualTestName1: { + avg_first_5: 2000, + avg_next_10: 1500, + }, + testSuiteName2_individualTestName2: { + avg_first_5: 2000, + avg_next_10: 1500, + }, + }); + }); + + it('should return an empty object when no results are found', async () => { + // Given + const suiteAndTestNamePairs = [ + { + testSuiteName: 'testSuiteName1', + individualTestName: 'individualTestName1', + }, + ]; + + mockQuery.resolves([]); + + // When + const results = await getAverageLimitValuesFromDB(suiteAndTestNamePairs); + + // Then + expect(mockQuery.calledOnce).to.be.true; + expect(results).to.deep.equal({}); + }); + + it('should handle missing fields and default them to zero', async () => { + // Given + const suiteAndTestNamePairs = [ + { + testSuiteName: 'testSuiteName1', + individualTestName: 'individualTestName1', + }, + ]; + + const mockResults = [ + { + test_suite_name: 'testSuiteName1', + individual_test_name: 'individualTestName1', + avg_first_5: null, + avg_next_10: undefined, + }, + ]; + + mockQuery.resolves(mockResults); + + // When + const results = await getAverageLimitValuesFromDB(suiteAndTestNamePairs); + + // Then + expect(results).to.deep.equal({ + testSuiteName1_individualTestName1: { + avg_first_5: 0, + avg_next_10: 0, + }, + }); + }); + + it('should handle an empty suiteAndTestNamePairs array and return an empty object', async () => { + // Given + const suiteAndTestNamePairs: { + testSuiteName: string; + individualTestName: string; + }[] = []; + + // Simulate no results (empty array) + mockQuery.resolves([]); + + // When + const results = await getAverageLimitValuesFromDB(suiteAndTestNamePairs); + + // Then + expect(results).to.deep.equal({}); + }); + + it('should handle errors and return an empty object', async () => { + // Given + const suiteAndTestNamePairs = [ + { + testSuiteName: 'testSuiteName1', + individualTestName: 'individualTestName1', + }, + ]; + + mockQuery.rejects(new Error('Database error')); + + // When + const results = await getAverageLimitValuesFromDB(suiteAndTestNamePairs); + + // Then + expect(results).to.deep.equal({}); + }); + }); + + describe('saveAlerts', () => { + it('should save alert', async () => { + // Given + const saveStub: sinon.SinonStub = sinon.stub().resolvesArg(0); + connectionStub.resolves({ + manager: { save: saveStub }, + } as unknown as DataSource); + + const savedEntity = new UiTestResult(); + savedEntity.id = 1; + savedEntity.testSuiteName = 'suite'; + savedEntity.individualTestName = 'test'; + savedEntity.componentLoadTime = 10; + savedEntity.salesforceLoadTime = 20; + savedEntity.overallLoadTime = 30; + + const alert: UiAlert = new UiAlert(); + alert.testSuiteName = savedEntity.testSuiteName; + alert.individualTestName = savedEntity.individualTestName; + alert.componentLoadTimeDegraded = 2; + alert.alertType = 'normal'; + const results = [alert]; + + // When + const savedRecords = await saveAlerts([savedEntity], results); + + // Then + expect(saveStub).to.be.calledOnce; + expect(savedRecords).to.eql(results); + expect(savedRecords[0].uiTestResultId).to.equal(1); + }); + }); +}); diff --git a/test/database/uiTestResult.test.ts b/test/database/uiTestResult.test.ts index 392ea94..3406de7 100644 --- a/test/database/uiTestResult.test.ts +++ b/test/database/uiTestResult.test.ts @@ -15,16 +15,26 @@ import { } from '../../src/database/uiTestResult'; import { DataSource } from 'typeorm'; import { UiTestResult } from '../../src/database/entity/uiTestResult'; +import * as uiAlert from '../../src/services/result/uiAlert'; +import { UiAlert } from '../../src/database/entity/uiAlert'; +import * as alertInfo from '../../src/database/uiAlertInfo'; chai.use(sinonChai); describe('src/database/uiTestResult', () => { let connectionStub: SinonStub; let saveRecordsStub: SinonStub; + let generateValidAlertsStub: SinonStub; + let alertInfoStub: SinonStub; beforeEach(() => { connectionStub = sinon.stub(db, 'getConnection'); saveRecordsStub = sinon.stub(saveModule, 'saveRecords'); + generateValidAlertsStub = sinon.stub(uiAlert, 'generateValidAlerts'); + alertInfoStub = sinon.stub(alertInfo, 'saveAlerts'); + + alertInfoStub.resolvesArg(0); + alertInfoStub.resolvesArg(1); }); afterEach(() => { @@ -51,6 +61,7 @@ describe('src/database/uiTestResult', () => { savedEntity.overallLoadTime = 30; saveRecordsStub.resolves([savedEntity]); + generateValidAlertsStub.resolves([]); // When const result = await saveUiTestResult([dto]); @@ -66,6 +77,46 @@ describe('src/database/uiTestResult', () => { overallLoadTime: 30, }, ]); + expect(generateValidAlertsStub).to.be.calledOnce; + expect(alertInfoStub).to.not.be.called; + }); + + it('should save alerts if any valid alert entries are returned', async () => { + // Given + const dto: UiTestResultDTO = { + testSuiteName: 'suite', + individualTestName: 'test', + componentLoadTime: 10, + salesforceLoadTime: 20, + overallLoadTime: 30, + }; + + const savedEntity = new UiTestResult(); + savedEntity.id = 1; + savedEntity.testSuiteName = 'suite'; + savedEntity.individualTestName = 'test'; + savedEntity.componentLoadTime = 10; + savedEntity.salesforceLoadTime = 20; + savedEntity.overallLoadTime = 30; + + const alert: UiAlert = new UiAlert(); + alert.testSuiteName = savedEntity.testSuiteName; + alert.individualTestName = savedEntity.individualTestName; + alert.componentLoadTimeDegraded = 2; + alert.alertType = 'normal'; + + saveRecordsStub.resolves([savedEntity]); + generateValidAlertsStub.resolves([alert]); + + // When + await saveUiTestResult([dto]); + + // Then + expect(saveRecordsStub).to.be.calledOnce; + expect(generateValidAlertsStub).to.be.calledOnce; + expect(alertInfoStub).to.be.calledOnce; + expect(alertInfoStub.args[0][0][0].id).to.equal(savedEntity.id); + expect(alertInfoStub.args[0][1][0].alertType).to.equal('normal'); }); }); diff --git a/test/services/uiAlert.test.ts b/test/services/uiAlert.test.ts new file mode 100644 index 0000000..b2d380d --- /dev/null +++ b/test/services/uiAlert.test.ts @@ -0,0 +1,243 @@ +import * as sinon from 'sinon'; +import chai, { expect } from 'chai'; +import sinonChai from 'sinon-chai'; + +import { generateValidAlerts } from '../../src/services/result/uiAlert'; +import * as env from '../../src/shared/env'; +import * as uiAlertInfo from '../../src/database/uiAlertInfo'; + +import { UiTestResultDTO } from '../../src/database/uiTestResult'; +import { NORMAL, CRITICAL } from '../../src/shared/constants'; + +chai.use(sinonChai); + +const MOCK_TEST_DTO_BASE: UiTestResultDTO = { + testSuiteName: 'ComponentLoadSuite', + individualTestName: 'ComponentXLoadTime', + overallLoadTime: 1000, + componentLoadTime: 500, + salesforceLoadTime: 500, + alertInfo: undefined, +} as UiTestResultDTO; + +describe('generateValidAlerts', () => { + let sandbox: sinon.SinonSandbox; + let storeAlertsStub: sinon.SinonStub; + let getAveragesStub: sinon.SinonStub; + let normalThresholdStub: sinon.SinonStub; + let criticalThresholdStub: sinon.SinonStub; + + before(() => { + sandbox = sinon.createSandbox(); + + storeAlertsStub = sandbox.stub(env, 'shouldStoreUiAlerts'); + normalThresholdStub = sandbox.stub(env, 'getNormalComponentLoadThreshold'); + criticalThresholdStub = sandbox.stub( + env, + 'getCriticalComponentLoadThreshold' + ); + + getAveragesStub = sandbox.stub(uiAlertInfo, 'getAverageLimitValuesFromDB'); + }); + + beforeEach(() => { + sandbox.resetHistory(); + + storeAlertsStub.returns(true); + normalThresholdStub.returns('20'); + criticalThresholdStub.returns('50'); + getAveragesStub.resolves({}); + }); + + after(() => { + sandbox.restore(); + }); + + describe('alert eligibility filtering', () => { + it('should return empty array if global alerts are disabled and no override exists', async () => { + // Given + storeAlertsStub.returns(false); + + // When + const result = await generateValidAlerts([MOCK_TEST_DTO_BASE]); + + // Then + expect(result).to.be.an('array').that.is.empty; + expect(getAveragesStub).to.not.have.been.called; + }); + + it('should enable processing if a test explicitly enables alerts (override global disable)', async () => { + // Given + storeAlertsStub.returns(false); + const overriddenTest = { + ...MOCK_TEST_DTO_BASE, + alertInfo: { storeAlerts: true }, + } as UiTestResultDTO; + + getAveragesStub.resolves({}); + + // When + await generateValidAlerts([overriddenTest]); + + // Then + expect(getAveragesStub).to.have.been.calledOnce; + }); + + it('should skip processing if a test explicitly disables alerts (override global enable)', async () => { + // Given + storeAlertsStub.returns(true); + const disabledTest = { + ...MOCK_TEST_DTO_BASE, + alertInfo: { storeAlerts: false }, + } as UiTestResultDTO; + + // When + const results = await generateValidAlerts([disabledTest]); + + // Then + expect(results).to.be.an('array').that.is.empty; + expect(getAveragesStub).to.not.have.been.called; + }); + }); + + describe('alert generation logic', () => { + const avgFirst5 = 200; + + it('should generate a CRITICAL alert when degradation is greater than critical threshold (50)', async () => { + //Given + const avgNext10 = 100; + const mockAverages = { + ['ComponentLoadSuite_ComponentXLoadTime']: { + avg_first_5: avgFirst5, + avg_next_10: avgNext10, + }, + }; + getAveragesStub.resolves(mockAverages); + + // When + const results = await generateValidAlerts([MOCK_TEST_DTO_BASE]); + + // Then + expect(results).to.have.lengthOf(1); + expect(results[0].alertType).to.equal(CRITICAL); + expect(results[0].componentLoadTimeDegraded).to.equal(100); + }); + + it('should generate a NORMAL alert when degradation is between normal (20) and critical (50)', async () => { + //Given + const avgNext10 = 165; + const mockAverages = { + ['ComponentLoadSuite_ComponentXLoadTime']: { + avg_first_5: avgFirst5, + avg_next_10: avgNext10, + }, + }; + getAveragesStub.resolves(mockAverages); + + // When + const results = await generateValidAlerts([MOCK_TEST_DTO_BASE]); + + // Then + expect(results).to.have.lengthOf(1); + expect(results[0].alertType).to.equal(NORMAL); + expect(results[0].componentLoadTimeDegraded).to.equal(35); + }); + + it('should return NO alert when degradation is below the normal threshold (20)', async () => { + // Given + const avgNext10 = 185; + const mockAverages = { + ['ComponentLoadSuite_ComponentXLoadTime']: { + avg_first_5: avgFirst5, + avg_next_10: avgNext10, + }, + }; + getAveragesStub.resolves(mockAverages); + + // When + const results = await generateValidAlerts([MOCK_TEST_DTO_BASE]); + + // Then + expect(results).to.be.an('array').that.is.empty; + }); + + it('should return NO alert when historical average data is not found', async () => { + // Given + getAveragesStub.resolves({}); + + // When + const results = await generateValidAlerts([MOCK_TEST_DTO_BASE]); + + // Then + expect(results).to.be.an('array').that.is.empty; + }); + + it('should return NO alert if degradation is zero', async () => { + //Given + const avgNext10 = 200; + const mockAverages = { + ['ComponentLoadSuite_ComponentXLoadTime']: { + avg_first_5: avgFirst5, + avg_next_10: avgNext10, + }, + }; + getAveragesStub.resolves(mockAverages); + + // When + const results = await generateValidAlerts([MOCK_TEST_DTO_BASE]); + + // Then + expect(results).to.be.an('array').that.is.empty; + }); + }); + + it('should use test-level thresholds if provided, overriding global environment settings', async () => { + // Given + const testLevelNormal = 10; + const testLevelCritical = 30; + + const avgNext10 = 175; + const mockAverages = { + ['ComponentLoadSuite_ComponentXLoadTime']: { + avg_first_5: 200, + avg_next_10: avgNext10, + }, + }; + getAveragesStub.resolves(mockAverages); + + const testWithOverride = { + ...MOCK_TEST_DTO_BASE, + alertInfo: { + uiAlertThresholds: { + componentLoadTimeThresholdNormal: testLevelNormal, + componentLoadTimeThresholdCritical: testLevelCritical, + }, + }, + } as UiTestResultDTO; + + // When + const results = await generateValidAlerts([testWithOverride]); + + // Then + expect(results).to.have.lengthOf(1); + expect(results[0].alertType).to.equal(NORMAL); + expect(results[0].componentLoadTimeDegraded).to.equal(25); + expect(normalThresholdStub).to.not.have.been.called; + }); + + it('should handle errors during average fetching and return an empty array, logging the error', async () => { + // Given + const errorStub = sandbox.stub(console, 'error'); + getAveragesStub.rejects(new Error('Database connection failed')); + + // When + const results = await generateValidAlerts([MOCK_TEST_DTO_BASE]); + + // Then + expect(results).to.be.an('array').that.is.empty; + expect(getAveragesStub).to.have.been.calledOnce; + expect(errorStub).to.have.been.calledWith( + sinon.match('Error generating alerts') + ); + }); +}); diff --git a/test/shared/env.test.ts b/test/shared/env.test.ts index b4c16bb..fbbe1cd 100644 --- a/test/shared/env.test.ts +++ b/test/shared/env.test.ts @@ -600,4 +600,91 @@ describe('shared/env/index', () => { expect(customRanges.soql_ranges).to.be.an('array'); }); }); + + describe('shouldStoreUiAlerts', () => { + it('returns true when STORE_UI_ALERTS is true', () => { + // Given + process.env.STORE_UI_ALERTS = 'true'; + + // When + const shouldStoreUiAlerts = env.shouldStoreUiAlerts(); + + // Then + expect(shouldStoreUiAlerts).to.be.true; + }); + + it('returns false when STORE_UI_ALERTS is false', () => { + // Given + process.env.STORE_UI_ALERTS = 'false'; + + // When + const shouldStoreUiAlerts = env.shouldStoreUiAlerts(); + + // Then + expect(shouldStoreUiAlerts).to.be.false; + }); + + it('returns false when STORE_UI_ALERTS is undefined', () => { + // Given + process.env.STORE_UI_ALERTS = undefined; + + // When + const shouldStoreUiAlerts = env.shouldStoreUiAlerts(); + + // Then + expect(shouldStoreUiAlerts).to.be.false; + }); + }); + + describe('getNormalComponentLoadThreshold', () => { + it('returns NORMAL_COMPONENT_LOAD_THRESHOLD given value', () => { + // Given + process.env.NORMAL_COMPONENT_LOAD_THRESHOLD = '10'; + + // When + const normalComponentLoadThreshold = + env.getNormalComponentLoadThreshold(); + + // Then + expect(normalComponentLoadThreshold).to.eql('10'); + }); + + it('return default value when NORMAL_COMPONENT_LOAD_THRESHOLD is not set', () => { + // Given + delete process.env.NORMAL_COMPONENT_LOAD_THRESHOLD; + + // When + const normalComponentLoadThreshold = + env.getNormalComponentLoadThreshold(); + + // Then + expect(normalComponentLoadThreshold).to.eql('1000'); + }); + }); + + describe('getCriticalComponentLoadThreshold', () => { + it('returns CRITICAL_COMPONENT_LOAD_THRESHOLD given value', () => { + // Given + process.env.CRITICAL_COMPONENT_LOAD_THRESHOLD = '10'; + + // When + const criticalComponentLoadThreshold = + env.getCriticalComponentLoadThreshold(); + + // Then + expect(criticalComponentLoadThreshold).to.eql('10'); + }); + + it('return default value when CRITICAL_COMPONENT_LOAD_THRESHOLD is not set', () => { + // Given + delete process.env.CRITICAL_COMPONENT_LOAD_THRESHOLD; + + // When + const criticalComponentLoadThreshold = + env.getCriticalComponentLoadThreshold(); + + // Then + expect(criticalComponentLoadThreshold).to.eql('10000'); + }); + }); });