feat: aggregate operation diffs

package.json

@@ -27,13 +27,13 @@
     "update-lock-file": "update-lock-file @netcracker"
   },
   "dependencies": {
-    "@netcracker/qubership-apihub-api-unifier": "dev",
+    "@netcracker/qubership-apihub-api-unifier": "feature-performance-optimization",
     "@netcracker/qubership-apihub-json-crawl": "1.0.4",
     "fast-equals": "4.0.3"
   },
   "devDependencies": {
     "@netcracker/qubership-apihub-compatibility-suites": "dev",
-    "@netcracker/qubership-apihub-graphapi": "1.0.8",
+    "@netcracker/qubership-apihub-graphapi": "feature-performance-optimization",
     "@netcracker/qubership-apihub-npm-gitflow": "3.1.0",
     "@types/jest": "29.5.11",
     "@types/node": "20.11.6",

src/api.ts

@@ -16,7 +16,13 @@ import {
   SpecType,
   OpenApiSpecVersion,
 } from '@netcracker/qubership-apihub-api-unifier'
-import { DEFAULT_NORMALIZED_RESULT, DEFAULT_OPTION_DEFAULTS_META_KEY, DEFAULT_OPTION_ORIGINS_META_KEY, DIFF_META_KEY } from './core'
+import {
+  DIFFS_AGGREGATED_META_KEY,
+  DEFAULT_NORMALIZED_RESULT,
+  DEFAULT_OPTION_DEFAULTS_META_KEY,
+  DEFAULT_OPTION_ORIGINS_META_KEY,
+  DIFF_META_KEY,
+} from './core'
 
 function isOpenApiSpecVersion(specType: SpecType): specType is OpenApiSpecVersion {
   return specType === SPEC_TYPE_OPEN_API_30 || specType === SPEC_TYPE_OPEN_API_31
@@ -55,6 +61,7 @@ export function apiDiff(before: unknown, after: unknown, options: CompareOptions
     metaKey: DIFF_META_KEY,
     defaultsFlag: DEFAULT_OPTION_DEFAULTS_META_KEY,
     originsFlag: DEFAULT_OPTION_ORIGINS_META_KEY,
+    diffsAggregatedFlag: DIFFS_AGGREGATED_META_KEY,
     compareScope: COMPARE_SCOPE_ROOT,
     mergedJsoCache: createEvaluationCacheService(),
     diffUniquenessCache: createEvaluationCacheService(),

src/core/compare.ts

@@ -1,4 +1,11 @@
-import { anyArrayKeys, getNodeRules, JsonPath, syncCrawl, SyncCrawlHook } from '@netcracker/qubership-apihub-json-crawl'
+import {
+  anyArrayKeys,
+  getNodeRules,
+  JsonPath,
+  syncClone,
+  syncCrawl,
+  SyncCrawlHook,
+} from '@netcracker/qubership-apihub-json-crawl'
 
 import {
   ChainItem,
@@ -16,6 +23,7 @@ import { deepEqual } from 'fast-equals'
 import {
   AdapterContext,
   AdapterResolver,
+  AGGREGATE_DIFFS_HERE_RULE,
   CompareContext,
   CompareResult,
   CompareRule,
@@ -235,7 +243,7 @@ const useMergeFactory = (onDiff: DiffCallback, options: InternalCompareOptions):
 
     const beforeKey = unsafeKey ?? (isArray(beforeJso) ? +Object.keys(keyMap).pop()! : Object.keys(keyMap).pop())
     const afterKey = keyMap[beforeKey]
-    const mergeKey = isArray(mergedJso) && isNumber(beforeKey) ? beforeKey : afterKey//THIS IS VERY FRAGILE. Cause this logic duplicate this line mergedJsoValue[keyInMerge] = afterValue[keyInAfter]
+    const mergeKey = isArray(mergedJso) && isNumber(beforeKey) ? beforeKey : afterKey//THIS IS VERY FRAGILE. Cause this logic duplicate this line mergedJsoValue[keyInMerge] = afterValue[keyInAfter] #gitleaks:allow
 
     // skip if node was removed
     if (!(beforeKey in keyMap)) {
@@ -473,7 +481,7 @@ export const compare = (before: unknown, after: unknown, options: InternalCompar
     return {
       diffs: rawDiffs,
       ownerDiffEntry: undefined,
-      merged,
+      merged: aggregateDiffs(merged, options),
     }
   }
   const diffFlags = Symbol('diffs')
@@ -491,8 +499,67 @@ export const compare = (before: unknown, after: unknown, options: InternalCompar
   return {
     diffs: denormalizedDiffs,
     ownerDiffEntry: undefined,
-    merged,
+    merged: merged,
+  }
+}
+
+export interface AggregateDiffsCrawlState {
+  operationDiffs?: Set<Diff>
+}
+
+export function aggregateDiffs(merged: unknown, options: InternalCompareOptions): unknown {
+  let activeDataCycleGuard: Set<unknown> = new Set()
+
+  const collectCurrentNodeDiffs = (value: Record<string | symbol, unknown>, operationDiffs: Set<Diff>) => {
+    if (options.metaKey in value) {
+      const diffs = value[options.metaKey] as Record<PropertyKey, unknown> | undefined
+      for (const key in diffs) {
+        operationDiffs.add(diffs[key] as Diff)
+      }
+    }
   }
+
+  syncClone<AggregateDiffsCrawlState>(
+    merged,
+    [
+      ({ key, value, state, rules }) => {
+        if (!isObject(value)) {
+          return { value }
+        }
+        if (typeof key === 'symbol') {
+          return { done: true }
+        }
+        if (activeDataCycleGuard.has(value)) {
+          return { done: true }
+        }
+        activeDataCycleGuard.add(value)
+
+        if (state.operationDiffs) {
+          collectCurrentNodeDiffs(value, state.operationDiffs)
+        }
+
+        if (rules && AGGREGATE_DIFFS_HERE_RULE in rules) {
+          activeDataCycleGuard = new Set()
+          const operationDiffs = new Set<Diff>()
+          collectCurrentNodeDiffs(value, operationDiffs)
+          return {
+            value,
+            state: { ...state, operationDiffs },
+            exitHook: () => {
+              value[options.diffsAggregatedFlag] = operationDiffs
+            },
+          }
+        }
+        return { value }
+      },
+    ],
+    {
+      state: {},
+      rules: options.rules,
+    },
+  )
+
+  return merged
 }
 
 export const nestedCompare = (before: unknown, after: unknown, options: InternalCompareOptions): CompareResult => {

src/core/constants.ts

@@ -1,6 +1,7 @@
 import { ClassifyRule } from '../types'
 
 export const DIFF_META_KEY = Symbol('$diff')
+export const DIFFS_AGGREGATED_META_KEY = Symbol('$diffs-aggregated')
 export const DEFAULT_NORMALIZED_RESULT = false
 export const DEFAULT_OPTION_DEFAULTS_META_KEY = Symbol('$defaults')
 export const DEFAULT_OPTION_ORIGINS_META_KEY = Symbol('$origins')

src/graphapi/graphapi.rules.ts

@@ -18,7 +18,7 @@ import {
   deepEqualsUniqueItemsArrayMappingResolver
 } from '../core'
 import { resolveSchemaDescriptionTemplates } from '../jsonSchema'
-import type { ClassifyRule, CompareRules, DescriptionTemplates, MappingResolver } from '../types'
+import { AGGREGATE_DIFFS_HERE_RULE, ClassifyRule, CompareRules, DescriptionTemplates, MappingResolver } from '../types'
 import { graphApiSchemaAdapter as graphApiTypeAdapter, removeNotCorrectlySupportedInterfacesAdapter } from './graphapi.adapter'
 import { COMPARE_SCOPE_COMPONENTS, COMPARE_SCOPE_DIRECTIVE_USAGES, COMPARE_SCOPE_ARGS, COMPARE_SCOPE_OUTPUT } from './graphapi.const'
 import { complexTypeCompareResolver } from './graphapi.resolver'
@@ -335,18 +335,21 @@ export const graphApiRules = (): CompareRules => {
     '/queries': {
       '/*': {
         ...methodRules,
+        [AGGREGATE_DIFFS_HERE_RULE]: true,
         $: addNonBreaking
       },
     },
     '/mutations': {
       '/*': {
         ...methodRules,
+        [AGGREGATE_DIFFS_HERE_RULE]: true,
         $: addNonBreaking
       },
     },
     '/subscriptions': {
       '/*': {
         ...methodRules,
+        [AGGREGATE_DIFFS_HERE_RULE]: true,
         $: addNonBreaking
       }
     },

src/index.ts

@@ -1,7 +1,16 @@
 export { COMPARE_MODE_DEFAULT, COMPARE_MODE_OPERATION } from './types'
 
 export {
-  ClassifierType, DiffAction, DIFF_META_KEY, breaking, nonBreaking, unclassified, annotation, deprecated, risky,
+  ClassifierType,
+  DiffAction,
+  DIFF_META_KEY,
+  DIFFS_AGGREGATED_META_KEY,
+  breaking,
+  nonBreaking,
+  unclassified,
+  annotation,
+  deprecated,
+  risky,
 } from './core'
 
 export { apiDiff } from './api'

src/openapi/openapi3.classify.ts

@@ -5,8 +5,8 @@ import {
   breakingIfAfterTrue,
   nonBreaking,
   PARENT_JUMP,
-  strictResolveValueFromContext,
   reverseClassifyRule,
+  strictResolveValueFromContext,
   transformClassifyRule,
   unclassified,
 } from '../core'
@@ -143,9 +143,12 @@ export const pathChangeClassifyRule: ClassifyRule = [
   ({ before, after }) => {
     const beforePath = before.key as string
     const afterPath = after.key as string
+    // todo uncomment
+    // const unifiedBeforePath = createPathUnifier(before)(beforePath)
+    // const unifiedAfterPath = createPathUnifier(after)(afterPath)
     const unifiedBeforePath = hidePathParamNames(beforePath)
     const unifiedAfterPath = hidePathParamNames(afterPath)
-    
+
     // If unified paths are the same, it means only parameter names changed
     return unifiedBeforePath === unifiedAfterPath ? annotation : breaking
   }

src/openapi/openapi3.mapping.ts

@@ -1,6 +1,14 @@
-import type { MapKeysResult, MappingResolver } from '../types'
-import { getStringValue, objectKeys, onlyExistedArrayIndexes } from '../utils'
+import { MapKeysResult, MappingResolver, NodeContext } from '../types'
+import {
+  difference,
+  getStringValue,
+  intersection,
+  objectKeys,
+  onlyExistedArrayIndexes,
+  removeSlashes,
+} from '../utils'
 import { mapPathParams } from './openapi3.utils'
+import { OpenAPIV3 } from 'openapi-types'
 
 export const singleOperationPathMappingResolver: MappingResolver<string> = (before, after) => {
 
@@ -23,32 +31,40 @@ export const singleOperationPathMappingResolver: MappingResolver<string> = (befo
   return result
 }
 
-export const pathMappingResolver: MappingResolver<string> = (before, after) => {
+export const pathMappingResolver: MappingResolver<string> = (before, after, ctx) => {
 
   const result: MapKeysResult<string> = { added: [], removed: [], mapped: {} }
 
-  const originalBeforeKeys = objectKeys(before)
-  const originalAfterKeys = objectKeys(after)
-  const unifiedAfterKeys = originalAfterKeys.map(hidePathParamNames)
+  const unifyBeforePath = createPathUnifier(ctx.before)
+  const unifyAfterPath = createPathUnifier(ctx.after)
 
-  const notMappedAfterIndices = new Set(originalAfterKeys.keys())
+  const unifiedBeforeKeyToKey = Object.fromEntries(objectKeys(before).map(key => [unifyBeforePath(key), key]))
+  const unifiedAfterKeyToKey = Object.fromEntries(objectKeys(after).map(key => [unifyAfterPath(key), key]))
 
-  originalBeforeKeys.forEach(beforeKey => {
-    const unifiedBeforePath = hidePathParamNames(beforeKey)
-    const index = unifiedAfterKeys.indexOf(unifiedBeforePath)
+  const unifiedBeforeKeys = Object.keys(unifiedBeforeKeyToKey)
+  const unifiedAfterKeys = Object.keys(unifiedAfterKeyToKey)
 
-    if (index < 0) {
-      // removed item
-      result.removed.push(beforeKey)
-    } else {
-      // mapped items
-      result.mapped[beforeKey] = originalAfterKeys[index]
-      notMappedAfterIndices.delete(index)
-    }
-  })
+  result.added = difference(unifiedAfterKeys, unifiedBeforeKeys).map(key => unifiedAfterKeyToKey[key])
+  result.removed = difference(unifiedBeforeKeys, unifiedAfterKeys).map(key => unifiedBeforeKeyToKey[key])
+  result.mapped = Object.fromEntries(
+    intersection(unifiedBeforeKeys, unifiedAfterKeys).map(key => [unifiedBeforeKeyToKey[key], unifiedAfterKeyToKey[key]]),
+  )
+
+  return result
+}
+
+export const methodMappingResolver: MappingResolver<string> = (before, after) => {
+
+  const result: MapKeysResult<string> = { added: [], removed: [], mapped: {} }
+
+  const beforeKeys = objectKeys(before)
+  const afterKeys = objectKeys(after)
 
-  // added items
-  notMappedAfterIndices.forEach((notMappedIndex) => result.added.push(originalAfterKeys[notMappedIndex]))
+  result.added = difference(afterKeys, beforeKeys)
+  result.removed = difference(beforeKeys, afterKeys)
+
+  const mapped = intersection(beforeKeys, afterKeys)
+  mapped.forEach(key => result.mapped[key] = key)
 
   return result
 }
@@ -175,6 +191,31 @@ function isWildcardCompatible(beforeType: string, afterType: string): boolean {
   return true
 }
 
+// todo copy-paste from api-processor
+export const extractOperationBasePath = (servers?: OpenAPIV3.ServerObject[]): string => {
+  if (!Array.isArray(servers) || !servers.length) { return '' }
+
+  try {
+    const [firstServer] = servers
+    let serverUrl = firstServer.url
+    const { variables = {} } = firstServer
+
+    for (const param of Object.keys(variables)) {
+      serverUrl = serverUrl.replace(new RegExp(`{${param}}`, 'g'), variables[param].default)
+    }
+
+    const { pathname } = new URL(serverUrl, 'https://localhost')
+    return pathname.slice(-1) === '/' ? pathname.slice(0, -1) : pathname
+  } catch (error) {
+    return ''
+  }
+}
+
+export function createPathUnifier(nodeContext: NodeContext): (path: string) => string {
+  const serverPrefix = extractOperationBasePath((nodeContext.root as OpenAPIV3.Document).servers) // /api/v2
+  return (path) => removeSlashes(`${serverPrefix}${hidePathParamNames(path)}`)
+}
+
 export function hidePathParamNames(path: string): string {
   return path.replace(PATH_PARAMETER_REGEXP, PATH_PARAM_UNIFIED_PLACEHOLDER)
 }

src/openapi/openapi3.rules.ts

@@ -30,6 +30,7 @@ import {
   deepEqualsUniqueItemsArrayMappingResolver,
 } from '../core'
 import {
+  AGGREGATE_DIFFS_HERE_RULE,
   COMPARE_MODE_OPERATION,
   CompareRules,
   DescriptionTemplates,
@@ -54,6 +55,7 @@ import {
 } from './openapi3.classify'
 import {
   contentMediaTypeMappingResolver,
+  methodMappingResolver,
   paramMappingResolver,
   pathMappingResolver,
   singleOperationPathMappingResolver,
@@ -345,6 +347,7 @@ export const openApi3Rules = (options: OpenApi3RulesOptions): CompareRules => {
 
   const operationRule: CompareRules = {
     $: [nonBreaking, breaking, unclassified],
+    [AGGREGATE_DIFFS_HERE_RULE]: true,
     '/callbacks': {
       '/*': {
         //no support?
@@ -390,7 +393,7 @@ export const openApi3Rules = (options: OpenApi3RulesOptions): CompareRules => {
 
   const pathItemObjectRules = (options: OpenApi3RulesOptions): CompareRules => ({
     $: pathChangeClassifyRule,
-    mapping: options.mode === COMPARE_MODE_OPERATION ? singleOperationPathMappingResolver : pathMappingResolver,
+    mapping: options.mode === COMPARE_MODE_OPERATION ? singleOperationPathMappingResolver : methodMappingResolver,
     '/description': { $: allAnnotation },
     '/parameters': {
       $: [nonBreaking, breaking, breaking],
@@ -431,6 +434,7 @@ export const openApi3Rules = (options: OpenApi3RulesOptions): CompareRules => {
       '/*': pathItemObjectRules(options),
     },
     '/securitySchemes': {
+      [AGGREGATE_DIFFS_HERE_RULE]: true,
       $: [breaking, nonBreaking, breaking],
       '/*': {
         $: [breaking, nonBreaking, breaking],
@@ -452,14 +456,18 @@ export const openApi3Rules = (options: OpenApi3RulesOptions): CompareRules => {
       ...documentAnnotationRule,
       '/**': documentAnnotationRule,
     },
-    '/servers': serversRules,
+    '/servers': {
+      [AGGREGATE_DIFFS_HERE_RULE]: true,
+      ...serversRules,
+    },
     '/paths': {
       $: allUnclassified,
       mapping: options.mode === COMPARE_MODE_OPERATION ? singleOperationPathMappingResolver : pathMappingResolver,
       '/*': pathItemObjectRules(options),
     },
     '/components': componentsRule,
     '/security': {
+      [AGGREGATE_DIFFS_HERE_RULE]: true,
       $: globalSecurityClassifyRule,
       '/*': { $: globalSecurityItemClassifyRule },
     },