diff --git a/CHANGELOG.md b/CHANGELOG.md index c05ed47..e5f3491 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Benchmarker - Changelog +## 6.0.1 + +- Update logic for UI test alerts to use 15 days worth of results for averages. +- Update logic for UI test alerts so that tests with less than 15 days worth of results will not alert. + ## 6.0.0 ### Breaking Changes diff --git a/package-lock.json b/package-lock.json index 98d35ec..e583e4a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@apexdevtools/benchmarker", - "version": "6.0.0", + "version": "6.0.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@apexdevtools/benchmarker", - "version": "6.0.0", + "version": "6.0.1", "dependencies": { "@jsforce/jsforce-node": "3.2.0", "@salesforce/core": "7.3.6", diff --git a/package.json b/package.json index e6bce26..a9a54ea 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@apexdevtools/benchmarker", - "version": "6.0.0", + "version": "6.0.1", "description": "Benchmarks performance of processes on Salesforce Orgs", "author": { "name": "Apex Dev Tools Team", @@ -25,7 +25,6 @@ "doc:generate": "typedoc --out docs/api ./src", "lint": "eslint ./src ./test ./test_system --fix", "prepare": "ts-patch install -s && husky", - "postversion": "npm run doc:generate", "test": "nyc mocha --timeout 10000 'test/**/*.test.*'", "test:only": "nyc mocha --timeout 10000", "test:system:init": "cp test_system/.env .env", diff --git a/src/database/uiAlertInfo.ts b/src/database/uiAlertInfo.ts index 6522308..435d958 100644 --- a/src/database/uiAlertInfo.ts +++ b/src/database/uiAlertInfo.ts @@ -30,57 +30,94 @@ export async function getAverageLimitValuesFromDB( ) { const connection = await getConnection(); + const countQuery = ` + SELECT individual_test_name, + COUNT(create_date_time) AS count_older_than_15_days + FROM performance.ui_test_result + WHERE create_date_time <= CURRENT_DATE - INTERVAL '15 days' + GROUP BY individual_test_name + `; + + const countResultMap: { + [key: string]: { count_older_than_15_days: number }; + } = {}; + try { + const countResult = await connection.query(countQuery); + countResult.forEach( + (row: { + individual_test_name: string; + count_older_than_15_days: number; + }) => { + countResultMap[row.individual_test_name] = { + count_older_than_15_days: row.count_older_than_15_days, + }; + } + ); + } catch (error) { + console.error('Error in fetching the count values: ', error); + return {}; + } + const suiteAndTestNameConditions = suiteAndTestNamePairs - .map(pair => `('${pair.testSuiteName}', '${pair.individualTestName}')`) + .flatMap(pair => { + if ( + countResultMap[pair.individualTestName]?.count_older_than_15_days > 0 + ) { + return [`('${pair.testSuiteName}', '${pair.individualTestName}')`]; + } + return []; + }) .join(', '); - const query = ` - WITH ranked AS ( - SELECT - test_suite_name, - individual_test_name, - component_load_time, - create_date_time, - ROW_NUMBER() OVER ( - PARTITION BY test_suite_name, individual_test_name - ORDER BY create_date_time DESC - ) AS rn - FROM performance.ui_test_result - WHERE (create_date_time >= CURRENT_TIMESTAMP - INTERVAL '30 DAYS') - AND (test_suite_name, individual_test_name) IN (${suiteAndTestNameConditions}) - ) - SELECT - test_suite_name, - individual_test_name, - ROUND(AVG(CASE WHEN rn BETWEEN 1 AND 5 THEN component_load_time END)::numeric, 0) AS avg_first_5, - ROUND(AVG(CASE WHEN rn BETWEEN 6 AND 15 THEN component_load_time END)::numeric, 0) AS avg_next_10 - FROM ranked - GROUP BY test_suite_name, individual_test_name - HAVING COUNT(*) >= 15 + if (suiteAndTestNameConditions.length === 0) { + return {}; + } + + const avgQuery = ` + SELECT + individual_test_name, + test_suite_name, + AVG(CASE + WHEN create_date_time >= CURRENT_DATE - INTERVAL '5 days' + THEN component_load_time + ELSE NULL + END) AS avg_load_time_past_5_days, + AVG(CASE + WHEN create_date_time >= CURRENT_DATE - INTERVAL '15 days' + AND create_date_time < CURRENT_DATE - INTERVAL '5 days' + THEN component_load_time + ELSE NULL + END) AS avg_load_time_6_to_15_days_ago + FROM performance.ui_test_result + WHERE create_date_time >= CURRENT_DATE - INTERVAL '15 days' + AND (test_suite_name, individual_test_name) IN (${suiteAndTestNameConditions}) + GROUP BY individual_test_name, test_suite_name + ORDER BY individual_test_name; `; const resultsMap: { [key: string]: { - avg_first_5: number; - avg_next_10: number; + avg_load_time_past_5_days: number; + avg_load_time_6_to_15_days_ago: number; }; } = {}; try { - const result = await connection.query(query); + const result = await connection.query(avgQuery); // Populate the results map result.forEach( (row: { test_suite_name: string; individual_test_name: string; - avg_first_5: number; - avg_next_10: number; + avg_load_time_past_5_days: number; + avg_load_time_6_to_15_days_ago: number; }) => { const key = `${row.test_suite_name}_${row.individual_test_name}`; resultsMap[key] = { - avg_first_5: row.avg_first_5 ?? 0, - avg_next_10: row.avg_next_10 ?? 0, + avg_load_time_past_5_days: row.avg_load_time_past_5_days ?? 0, + avg_load_time_6_to_15_days_ago: + row.avg_load_time_6_to_15_days_ago ?? 0, }; } ); diff --git a/src/services/result/uiAlert.ts b/src/services/result/uiAlert.ts index 8dc001a..8b0d4f6 100644 --- a/src/services/result/uiAlert.ts +++ b/src/services/result/uiAlert.ts @@ -57,8 +57,8 @@ async function addAlertByComparingAvg( output: UiTestResultDTO, preFetchedAverages: { [key: string]: { - avg_first_5: number; - avg_next_10: number; + avg_load_time_past_5_days: number; + avg_load_time_6_to_15_days_ago: number; }; } ): Promise { @@ -85,7 +85,8 @@ async function addAlertByComparingAvg( ? output.alertInfo.uiAlertThresholds.componentLoadTimeThresholdCritical : Number(getCriticalComponentLoadThreshold()); const componentLoadThresholdDegraded = Math.abs( - averageResults.avg_first_5 - averageResults.avg_next_10 + averageResults.avg_load_time_past_5_days - + averageResults.avg_load_time_6_to_15_days_ago ); if ( diff --git a/test/database/uiAlertInfo.test.ts b/test/database/uiAlertInfo.test.ts index 32f7ebe..61b62b9 100644 --- a/test/database/uiAlertInfo.test.ts +++ b/test/database/uiAlertInfo.test.ts @@ -16,7 +16,7 @@ import { UiTestResult } from '../../src/database/entity/uiTestResult'; chai.use(sinonChai); -describe('src/database/alertInfo', () => { +describe('src/database/uiAlertInfo', () => { let mockQuery: sinon.SinonStub; let connectionStub: sinon.SinonStub; let mockDataSource: any; @@ -45,61 +45,174 @@ describe('src/database/alertInfo', () => { }, ]; - const mockResults = [ + const mockCountResults = [ + { + individual_test_name: 'individualTestName1', + count_older_than_15_days: 20, + }, + { + individual_test_name: 'individualTestName2', + count_older_than_15_days: 18, + }, + ]; + + const mockAvgResults = [ { test_suite_name: 'testSuiteName1', individual_test_name: 'individualTestName1', - avg_first_5: 2000, - avg_next_10: 1500, + avg_load_time_past_5_days: 2000, + avg_load_time_6_to_15_days_ago: 1500, }, { test_suite_name: 'testSuiteName2', individual_test_name: 'individualTestName2', - avg_first_5: 2000, - avg_next_10: 1500, + avg_load_time_past_5_days: 2000, + avg_load_time_6_to_15_days_ago: 1500, }, ]; - mockQuery.resolves(mockResults); + mockQuery.onFirstCall().resolves(mockCountResults); + mockQuery.onSecondCall().resolves(mockAvgResults); // When const results = await getAverageLimitValuesFromDB(suiteAndTestNamePairs); // Then - expect(mockQuery.calledOnce).to.be.true; - expect(mockQuery.args[0][0]).to.include('SELECT'); - expect(mockQuery.args[0][0]).to.include( + expect(mockQuery.calledTwice).to.be.true; + expect(mockQuery.args[1][0]).to.include('SELECT'); + expect(mockQuery.args[1][0]).to.include( "(test_suite_name, individual_test_name) IN (('testSuiteName1', 'individualTestName1'), ('testSuiteName2', 'individualTestName2'))" ); expect(results).to.deep.equal({ testSuiteName1_individualTestName1: { - avg_first_5: 2000, - avg_next_10: 1500, + avg_load_time_past_5_days: 2000, + avg_load_time_6_to_15_days_ago: 1500, }, testSuiteName2_individualTestName2: { - avg_first_5: 2000, - avg_next_10: 1500, + avg_load_time_past_5_days: 2000, + avg_load_time_6_to_15_days_ago: 1500, }, }); }); - it('should return an empty object when no results are found', async () => { + it('should not return average limit values when a test has no results older than 15 days', async () => { // Given const suiteAndTestNamePairs = [ { testSuiteName: 'testSuiteName1', individualTestName: 'individualTestName1', }, + { + testSuiteName: 'testSuiteName2', + individualTestName: 'individualTestName2', + }, + ]; + + const mockCountResults = [ + { + individual_test_name: 'individualTestName1', + count_older_than_15_days: 20, + }, + { + individual_test_name: 'individualTestName2', + count_older_than_15_days: 0, + }, + ]; + + const mockAvgResults = [ + { + test_suite_name: 'testSuiteName1', + individual_test_name: 'individualTestName1', + avg_load_time_past_5_days: 2000, + avg_load_time_6_to_15_days_ago: 1500, + }, ]; - mockQuery.resolves([]); + mockQuery.onFirstCall().resolves(mockCountResults); + mockQuery.onSecondCall().resolves(mockAvgResults); + + // When + const results = await getAverageLimitValuesFromDB(suiteAndTestNamePairs); + + // Then + expect(mockQuery.calledTwice).to.be.true; + expect(mockQuery.args[1][0]).to.include('SELECT'); + expect(mockQuery.args[1][0]).to.include( + "(test_suite_name, individual_test_name) IN (('testSuiteName1', 'individualTestName1'))" + ); + + expect(results).to.deep.equal({ + testSuiteName1_individualTestName1: { + avg_load_time_past_5_days: 2000, + avg_load_time_6_to_15_days_ago: 1500, + }, + }); + }); + + it('should not return average limit values when no results older than 15 days', async () => { + // Given + const suiteAndTestNamePairs = [ + { + testSuiteName: 'testSuiteName1', + individualTestName: 'individualTestName1', + }, + { + testSuiteName: 'testSuiteName2', + individualTestName: 'individualTestName2', + }, + ]; + + const mockCountResults = [ + { + individual_test_name: 'individualTestName1', + count_older_than_15_days: 0, + }, + { + individual_test_name: 'individualTestName2', + count_older_than_15_days: 0, + }, + ]; + + mockQuery.onFirstCall().resolves(mockCountResults); // When const results = await getAverageLimitValuesFromDB(suiteAndTestNamePairs); // Then expect(mockQuery.calledOnce).to.be.true; + + expect(results).to.deep.equal({}); + }); + + it('should return an empty object when no results are found', async () => { + // Given + const suiteAndTestNamePairs = [ + { + testSuiteName: 'testSuiteName1', + individualTestName: 'individualTestName1', + }, + ]; + + const mockCountResults = [ + { + individual_test_name: 'individualTestName1', + count_older_than_15_days: 20, + }, + { + individual_test_name: 'individualTestName2', + count_older_than_15_days: 18, + }, + ]; + + mockQuery.onFirstCall().resolves(mockCountResults); + mockQuery.onSecondCall().resolves([]); + + // When + const results = await getAverageLimitValuesFromDB(suiteAndTestNamePairs); + + // Then + expect(mockQuery.calledTwice).to.be.true; expect(results).to.deep.equal({}); }); @@ -112,16 +225,28 @@ describe('src/database/alertInfo', () => { }, ]; - const mockResults = [ + const mockCountResults = [ + { + individual_test_name: 'individualTestName1', + count_older_than_15_days: 20, + }, + { + individual_test_name: 'individualTestName2', + count_older_than_15_days: 18, + }, + ]; + + const mockAvgResults = [ { test_suite_name: 'testSuiteName1', individual_test_name: 'individualTestName1', - avg_first_5: null, - avg_next_10: undefined, + avg_load_time_past_5_days: null, + avg_load_time_6_to_15_days_ago: undefined, }, ]; - mockQuery.resolves(mockResults); + mockQuery.onFirstCall().resolves(mockCountResults); + mockQuery.onSecondCall().resolves(mockAvgResults); // When const results = await getAverageLimitValuesFromDB(suiteAndTestNamePairs); @@ -129,8 +254,8 @@ describe('src/database/alertInfo', () => { // Then expect(results).to.deep.equal({ testSuiteName1_individualTestName1: { - avg_first_5: 0, - avg_next_10: 0, + avg_load_time_past_5_days: 0, + avg_load_time_6_to_15_days_ago: 0, }, }); }); @@ -143,7 +268,8 @@ describe('src/database/alertInfo', () => { }[] = []; // Simulate no results (empty array) - mockQuery.resolves([]); + mockQuery.onFirstCall().resolves([]); + mockQuery.onSecondCall().resolves([]); // When const results = await getAverageLimitValuesFromDB(suiteAndTestNamePairs); diff --git a/test/services/uiAlert.test.ts b/test/services/uiAlert.test.ts index b2d380d..9870a0a 100644 --- a/test/services/uiAlert.test.ts +++ b/test/services/uiAlert.test.ts @@ -108,8 +108,8 @@ describe('generateValidAlerts', () => { const avgNext10 = 100; const mockAverages = { ['ComponentLoadSuite_ComponentXLoadTime']: { - avg_first_5: avgFirst5, - avg_next_10: avgNext10, + avg_load_time_past_5_days: avgFirst5, + avg_load_time_6_to_15_days_ago: avgNext10, }, }; getAveragesStub.resolves(mockAverages); @@ -128,8 +128,8 @@ describe('generateValidAlerts', () => { const avgNext10 = 165; const mockAverages = { ['ComponentLoadSuite_ComponentXLoadTime']: { - avg_first_5: avgFirst5, - avg_next_10: avgNext10, + avg_load_time_past_5_days: avgFirst5, + avg_load_time_6_to_15_days_ago: avgNext10, }, }; getAveragesStub.resolves(mockAverages); @@ -148,8 +148,8 @@ describe('generateValidAlerts', () => { const avgNext10 = 185; const mockAverages = { ['ComponentLoadSuite_ComponentXLoadTime']: { - avg_first_5: avgFirst5, - avg_next_10: avgNext10, + avg_load_time_past_5_days: avgFirst5, + avg_load_time_6_to_15_days_ago: avgNext10, }, }; getAveragesStub.resolves(mockAverages); @@ -177,8 +177,8 @@ describe('generateValidAlerts', () => { const avgNext10 = 200; const mockAverages = { ['ComponentLoadSuite_ComponentXLoadTime']: { - avg_first_5: avgFirst5, - avg_next_10: avgNext10, + avg_load_time_past_5_days: avgFirst5, + avg_load_time_6_to_15_days_ago: avgNext10, }, }; getAveragesStub.resolves(mockAverages); @@ -199,8 +199,8 @@ describe('generateValidAlerts', () => { const avgNext10 = 175; const mockAverages = { ['ComponentLoadSuite_ComponentXLoadTime']: { - avg_first_5: 200, - avg_next_10: avgNext10, + avg_load_time_past_5_days: 200, + avg_load_time_6_to_15_days_ago: avgNext10, }, }; getAveragesStub.resolves(mockAverages);