From 672c08dc2added7224728d0a9e2df555730d3cde Mon Sep 17 00:00:00 2001 From: Connor Lien Date: Mon, 30 May 2022 14:53:27 -0700 Subject: [PATCH 1/9] Implement PIDS Lexer --- add-in/package-lock.json | 661 ++++++++---------- add-in/package.json | 2 + add-in/src/api/pids/archive/lexer.ts | 320 +++++++++ add-in/src/api/pids/archive/parsingTest.ts | 90 +++ add-in/src/api/pids/archive/testTokens.txt | 15 + add-in/src/api/pids/pattern/extractPattern.ts | 32 + add-in/src/api/pids/pattern/pattern.ts | 27 + add-in/src/api/pids/pattern/patternMiner.ts | 13 + add-in/src/api/pids/tests/extractionTest.ts | 4 + add-in/src/api/pids/tests/testMatrix.ts | 17 + add-in/src/api/pids/tests/tokenizeTest.ts | 4 + add-in/src/api/pids/tokenize/lexer.ts | 82 +++ add-in/src/api/pids/tokenize/token.ts | 71 ++ add-in/src/api/pids/tokenize/tokenizer.ts | 11 + add-in/src/api/pids/utils/hash.ts | 12 + package-lock.json | 3 + 16 files changed, 1012 insertions(+), 352 deletions(-) create mode 100644 add-in/src/api/pids/archive/lexer.ts create mode 100644 add-in/src/api/pids/archive/parsingTest.ts create mode 100644 add-in/src/api/pids/archive/testTokens.txt create mode 100644 add-in/src/api/pids/pattern/extractPattern.ts create mode 100644 add-in/src/api/pids/pattern/pattern.ts create mode 100644 add-in/src/api/pids/pattern/patternMiner.ts create mode 100644 add-in/src/api/pids/tests/extractionTest.ts create mode 100644 add-in/src/api/pids/tests/testMatrix.ts create mode 100644 add-in/src/api/pids/tests/tokenizeTest.ts create mode 100644 add-in/src/api/pids/tokenize/lexer.ts create mode 100644 add-in/src/api/pids/tokenize/token.ts create mode 100644 add-in/src/api/pids/tokenize/tokenizer.ts create mode 100644 add-in/src/api/pids/utils/hash.ts create mode 100644 package-lock.json diff --git a/add-in/package-lock.json b/add-in/package-lock.json index d83b0c1..75a3885 100644 --- a/add-in/package-lock.json +++ b/add-in/package-lock.json @@ -23,38 +23,46 @@ } }, "@babel/compat-data": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.17.0.tgz", - "integrity": "sha512-392byTlpGWXMv4FbyWw3sAZ/FrW/DrwqLGXpy0mbyNe9Taqv1mg9yON5/o0cnr8XYCkFTZbC1eV+c+LAROgrng==", + "version": "7.17.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.17.7.tgz", + "integrity": "sha512-p8pdE6j0a29TNGebNm7NzYZWB3xVZJBZ7XGs42uAKzQo8VQ3F0By/cQCtUEABwIqw5zo6WA4NbmxsfzADzMKnQ==", "dev": true }, "@babel/core": { - "version": "7.17.5", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.17.5.tgz", - "integrity": "sha512-/BBMw4EvjmyquN5O+t5eh0+YqB3XXJkYD2cjKpYtWOfFy4lQ4UozNSmxAcWT8r2XtZs0ewG+zrfsqeR15i1ajA==", + "version": "7.17.9", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.17.9.tgz", + "integrity": "sha512-5ug+SfZCpDAkVp9SFIZAzlW18rlzsOcJGaetCjkySnrXXDUw9AR8cDUm1iByTmdWM6yxX6/zycaV76w3YTF2gw==", "dev": true, "requires": { "@ampproject/remapping": "^2.1.0", "@babel/code-frame": "^7.16.7", - "@babel/generator": "^7.17.3", - "@babel/helper-compilation-targets": "^7.16.7", - "@babel/helper-module-transforms": "^7.16.7", - "@babel/helpers": "^7.17.2", - "@babel/parser": "^7.17.3", + "@babel/generator": "^7.17.9", + "@babel/helper-compilation-targets": "^7.17.7", + "@babel/helper-module-transforms": "^7.17.7", + "@babel/helpers": "^7.17.9", + "@babel/parser": "^7.17.9", "@babel/template": "^7.16.7", - "@babel/traverse": "^7.17.3", + "@babel/traverse": "^7.17.9", "@babel/types": "^7.17.0", "convert-source-map": "^1.7.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", - "json5": "^2.1.2", + "json5": "^2.2.1", "semver": "^6.3.0" + }, + "dependencies": { + "json5": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", + "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "dev": true + } } }, "@babel/generator": { - "version": "7.17.3", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.17.3.tgz", - "integrity": "sha512-+R6Dctil/MgUsZsZAkYgK+ADNSZzJRRy0TvY65T71z/CR854xHQ1EweBYXdfT+HNeN7w0cSJJEzgxZMv40pxsg==", + "version": "7.17.9", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.17.9.tgz", + "integrity": "sha512-rAdDousTwxbIxbz5I7GEQ3lUip+xVCXooZNbsydCWs3xA7ZsYOv+CFRdzGxRX78BmQHu9B1Eso59AOZQOJDEdQ==", "dev": true, "requires": { "@babel/types": "^7.17.0", @@ -72,27 +80,27 @@ } }, "@babel/helper-compilation-targets": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.16.7.tgz", - "integrity": "sha512-mGojBwIWcwGD6rfqgRXVlVYmPAv7eOpIemUG3dGnDdCY4Pae70ROij3XmfrH6Fa1h1aiDylpglbZyktfzyo/hA==", + "version": "7.17.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.17.7.tgz", + "integrity": "sha512-UFzlz2jjd8kroj0hmCFV5zr+tQPi1dpC2cRsDV/3IEW8bJfCPrPpmcSN6ZS8RqIq4LXcmpipCQFPddyFA5Yc7w==", "dev": true, "requires": { - "@babel/compat-data": "^7.16.4", + "@babel/compat-data": "^7.17.7", "@babel/helper-validator-option": "^7.16.7", "browserslist": "^4.17.5", "semver": "^6.3.0" } }, "@babel/helper-create-class-features-plugin": { - "version": "7.17.6", - "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.17.6.tgz", - "integrity": "sha512-SogLLSxXm2OkBbSsHZMM4tUi8fUzjs63AT/d0YQIzr6GSd8Hxsbk2KYDX0k0DweAzGMj/YWeiCsorIdtdcW8Eg==", + "version": "7.17.9", + "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.17.9.tgz", + "integrity": "sha512-kUjip3gruz6AJKOq5i3nC6CoCEEF/oHH3cp6tOZhB+IyyyPyW0g1Gfsxn3mkk6S08pIA2y8GQh609v9G/5sHVQ==", "dev": true, "requires": { "@babel/helper-annotate-as-pure": "^7.16.7", "@babel/helper-environment-visitor": "^7.16.7", - "@babel/helper-function-name": "^7.16.7", - "@babel/helper-member-expression-to-functions": "^7.16.7", + "@babel/helper-function-name": "^7.17.9", + "@babel/helper-member-expression-to-functions": "^7.17.7", "@babel/helper-optimise-call-expression": "^7.16.7", "@babel/helper-replace-supers": "^7.16.7", "@babel/helper-split-export-declaration": "^7.16.7" @@ -108,23 +116,13 @@ } }, "@babel/helper-function-name": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.16.7.tgz", - "integrity": "sha512-QfDfEnIUyyBSR3HtrtGECuZ6DAyCkYFp7GHl75vFtTnn6pjKeK0T1DB5lLkFvBea8MdaiUABx3osbgLyInoejA==", + "version": "7.17.9", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.17.9.tgz", + "integrity": "sha512-7cRisGlVtiVqZ0MW0/yFB4atgpGLWEHUVYnb448hZK4x+vih0YO5UoS11XIYtZYqHd0dIPMdUSv8q5K4LdMnIg==", "dev": true, "requires": { - "@babel/helper-get-function-arity": "^7.16.7", "@babel/template": "^7.16.7", - "@babel/types": "^7.16.7" - } - }, - "@babel/helper-get-function-arity": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.16.7.tgz", - "integrity": "sha512-flc+RLSOBXzNzVhcLu6ujeHUrD6tANAOU5ojrRx/as+tbzf8+stUCj7+IfRRoAbEZqj/ahXEMsjhOhgeZsrnTw==", - "dev": true, - "requires": { - "@babel/types": "^7.16.7" + "@babel/types": "^7.17.0" } }, "@babel/helper-hoist-variables": { @@ -137,12 +135,12 @@ } }, "@babel/helper-member-expression-to-functions": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.16.7.tgz", - "integrity": "sha512-VtJ/65tYiU/6AbMTDwyoXGPKHgTsfRarivm+YbB5uAzKUyuPjgZSgAFeG87FCigc7KNHu2Pegh1XIT3lXjvz3Q==", + "version": "7.17.7", + "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.17.7.tgz", + "integrity": "sha512-thxXgnQ8qQ11W2wVUObIqDL4p148VMxkt5T/qpN5k2fboRyzFGFmKsTGViquyM5QHKUy48OZoca8kw4ajaDPyw==", "dev": true, "requires": { - "@babel/types": "^7.16.7" + "@babel/types": "^7.17.0" } }, "@babel/helper-module-imports": { @@ -155,14 +153,14 @@ } }, "@babel/helper-module-transforms": { - "version": "7.17.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.17.6.tgz", - "integrity": "sha512-2ULmRdqoOMpdvkbT8jONrZML/XALfzxlb052bldftkicAUy8AxSCkD5trDPQcwHNmolcl7wP6ehNqMlyUw6AaA==", + "version": "7.17.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.17.7.tgz", + "integrity": "sha512-VmZD99F3gNTYB7fJRDTi+u6l/zxY0BE6OIxPSU7a50s6ZUQkHwSDmV92FfM+oCG0pZRVojGYhkR8I0OGeCVREw==", "dev": true, "requires": { "@babel/helper-environment-visitor": "^7.16.7", "@babel/helper-module-imports": "^7.16.7", - "@babel/helper-simple-access": "^7.16.7", + "@babel/helper-simple-access": "^7.17.7", "@babel/helper-split-export-declaration": "^7.16.7", "@babel/helper-validator-identifier": "^7.16.7", "@babel/template": "^7.16.7", @@ -199,12 +197,12 @@ } }, "@babel/helper-simple-access": { - "version": "7.16.7", - "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.16.7.tgz", - "integrity": "sha512-ZIzHVyoeLMvXMN/vok/a4LWRy8G2v205mNP0XOuf9XRLyX5/u9CnVulUtDgUTama3lT+bf/UqucuZjqiGuTS1g==", + "version": "7.17.7", + "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.17.7.tgz", + "integrity": "sha512-txyMCGroZ96i+Pxr3Je3lzEJjqwaRC9buMUgtomcrLe5Nd0+fk1h0LLA+ixUF5OW7AhHuQ7Es1WcQJZmZsz2XA==", "dev": true, "requires": { - "@babel/types": "^7.16.7" + "@babel/types": "^7.17.0" } }, "@babel/helper-split-export-declaration": { @@ -229,20 +227,20 @@ "dev": true }, "@babel/helpers": { - "version": "7.17.2", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.17.2.tgz", - "integrity": "sha512-0Qu7RLR1dILozr/6M0xgj+DFPmi6Bnulgm9M8BVa9ZCWxDqlSnqt3cf8IDPB5m45sVXUZ0kuQAgUrdSFFH79fQ==", + "version": "7.17.9", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.17.9.tgz", + "integrity": "sha512-cPCt915ShDWUEzEp3+UNRktO2n6v49l5RSnG9M5pS24hA+2FAc5si+Pn1i4VVbQQ+jh+bIZhPFQOJOzbrOYY1Q==", "dev": true, "requires": { "@babel/template": "^7.16.7", - "@babel/traverse": "^7.17.0", + "@babel/traverse": "^7.17.9", "@babel/types": "^7.17.0" } }, "@babel/highlight": { - "version": "7.16.10", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.16.10.tgz", - "integrity": "sha512-5FnTQLSLswEj6IkgVw5KusNUUFY9ZGqe/TRFnP/BKYHYgfh7tc+C7mwiy95/yNP7Dh9x580Vv8r7u7ZfTBFxdw==", + "version": "7.17.9", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.17.9.tgz", + "integrity": "sha512-J9PfEKCbFIv2X5bjTMiZu6Vf341N05QIY+d6FvVKynkG1S7G0j3I0QoRtWIrXhZ+/Nlb5Q0MzqL7TokEJ5BNHg==", "dev": true, "requires": { "@babel/helper-validator-identifier": "^7.16.7", @@ -251,9 +249,9 @@ } }, "@babel/parser": { - "version": "7.17.3", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.17.3.tgz", - "integrity": "sha512-7yJPvPV+ESz2IUTPbOL+YkIGyCqOyNIzdguKQuJGnH7bg1WTIifuM21YqokFt/THWh1AkCRn9IgoykTRCBVpzA==", + "version": "7.17.9", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.17.9.tgz", + "integrity": "sha512-vqUSBLP8dQHFPdPi9bc5GK9vRkYHJ49fsZdtoJ8EQ8ibpwk5rPKfvNIwChB0KVXcIjcepEBBd2VHC5r9Gy8ueg==", "dev": true }, "@babel/plugin-syntax-typescript": { @@ -299,18 +297,18 @@ } }, "@babel/traverse": { - "version": "7.17.3", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.17.3.tgz", - "integrity": "sha512-5irClVky7TxRWIRtxlh2WPUUOLhcPN06AGgaQSB8AEwuyEBgJVuJ5imdHm5zxk8w0QS5T+tDfnDxAlhWjpb7cw==", + "version": "7.17.9", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.17.9.tgz", + "integrity": "sha512-PQO8sDIJ8SIwipTPiR71kJQCKQYB5NGImbOviK8K+kg5xkNSYXLBupuX9QhatFowrsvo9Hj8WgArg3W7ijNAQw==", "dev": true, "requires": { "@babel/code-frame": "^7.16.7", - "@babel/generator": "^7.17.3", + "@babel/generator": "^7.17.9", "@babel/helper-environment-visitor": "^7.16.7", - "@babel/helper-function-name": "^7.16.7", + "@babel/helper-function-name": "^7.17.9", "@babel/helper-hoist-variables": "^7.16.7", "@babel/helper-split-export-declaration": "^7.16.7", - "@babel/parser": "^7.17.3", + "@babel/parser": "^7.17.9", "@babel/types": "^7.17.0", "debug": "^4.1.0", "globals": "^11.1.0" @@ -327,9 +325,9 @@ } }, "@discoveryjs/json-ext": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.5.6.tgz", - "integrity": "sha512-ws57AidsDvREKrZKYffXddNkyaF14iHNHm8VQnZH6t99E8gczjNN0GpvcGny0imC80yQ0tHz1xVUKk/KFQSUyA==", + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz", + "integrity": "sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==", "dev": true }, "@eslint/eslintrc": { @@ -350,9 +348,9 @@ }, "dependencies": { "globals": { - "version": "13.12.1", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.12.1.tgz", - "integrity": "sha512-317dFlgY2pdJZ9rspXDks7073GpDmXdfbM3vYYp0HAMKGDh1FfWPleI2ljVNLQX5M5lXcAslTcPTrOrMEFOjyw==", + "version": "13.13.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.13.0.tgz", + "integrity": "sha512-EQ7Q18AJlPwp3vUDL4mKA0KXrXyNIQyWon6T6XQiBQF0XHvRsiCSrWmmeATpUzdJN2HhWZU6Pdl0a9zdep5p6A==", "dev": true, "requires": { "type-fest": "^0.20.2" @@ -367,9 +365,9 @@ } }, "@fluentui/date-time-utilities": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/@fluentui/date-time-utilities/-/date-time-utilities-8.4.0.tgz", - "integrity": "sha512-fEc8JMG0Y21De6kjD5xfP/ix7y3u0zbFTKgNWNASGpIYsZKXbJqeu8/DEqnc4x9Lf5pxiV+WhNcoMLbtjKTpmg==", + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/@fluentui/date-time-utilities/-/date-time-utilities-8.5.0.tgz", + "integrity": "sha512-SddqPNEA5PBxZLvRY9ej2//iTzNWFqBt9kZ9rjieBlRtFPjztnDV10Zq3xlR6ss79dwkiP+S+SP4SmI2xuckHA==", "requires": { "@fluentui/set-version": "^8.2.0", "tslib": "^2.1.0" @@ -399,12 +397,12 @@ } }, "@fluentui/font-icons-mdl2": { - "version": "8.2.1", - "resolved": "https://registry.npmjs.org/@fluentui/font-icons-mdl2/-/font-icons-mdl2-8.2.1.tgz", - "integrity": "sha512-cOogQx6KspcKkxr3D3Ghyd/QI6uywId4/PbOHGh2zbJpq2eqwCvQk+Hh6lDXnj6N4V2N2ph/+TxFNCsDL3QH2g==", + "version": "8.2.5", + "resolved": "https://registry.npmjs.org/@fluentui/font-icons-mdl2/-/font-icons-mdl2-8.2.5.tgz", + "integrity": "sha512-IgzcoOm+hZMlYJBR1RFb3fner1IxdSLCsnsjkVUtfnwuvGq/zjqEpNgsk5eUG4dHlu3E9dDY86nLqfGJe8R6yw==", "requires": { "@fluentui/set-version": "^8.2.0", - "@fluentui/style-utilities": "^8.6.1", + "@fluentui/style-utilities": "^8.6.5", "tslib": "^2.1.0" }, "dependencies": { @@ -416,14 +414,14 @@ } }, "@fluentui/foundation-legacy": { - "version": "8.2.1", - "resolved": "https://registry.npmjs.org/@fluentui/foundation-legacy/-/foundation-legacy-8.2.1.tgz", - "integrity": "sha512-id0PfYe+RYDlLfPqTIFElM1xSWtlQHAumsJUUiCh2pWbl3fURTfn/3UusqxB6rIeyyZtYPbI4Ex6am8hjDZkog==", + "version": "8.2.5", + "resolved": "https://registry.npmjs.org/@fluentui/foundation-legacy/-/foundation-legacy-8.2.5.tgz", + "integrity": "sha512-PP8Wbw1QK9lmzReh+C7XG7A7Q3WTXAaxAMHjcBE3m2OPnhlF77Ok6IAJBkuKTWg0SZYQqJilTDSkVDnXVylGgQ==", "requires": { - "@fluentui/merge-styles": "^8.4.0", + "@fluentui/merge-styles": "^8.5.0", "@fluentui/set-version": "^8.2.0", - "@fluentui/style-utilities": "^8.6.1", - "@fluentui/utilities": "^8.6.0", + "@fluentui/style-utilities": "^8.6.5", + "@fluentui/utilities": "^8.8.1", "tslib": "^2.1.0" }, "dependencies": { @@ -450,9 +448,9 @@ } }, "@fluentui/merge-styles": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/@fluentui/merge-styles/-/merge-styles-8.4.0.tgz", - "integrity": "sha512-hgo8MJRNwg2C6yNKsHKBhYb4Dq1oKA6fO99O4ee2p1HzAJ4HzbJScIe9pyqJaY4G4i2nCg6wH7+6RXyls3RanQ==", + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/@fluentui/merge-styles/-/merge-styles-8.5.0.tgz", + "integrity": "sha512-+WoaAaoYx/wfkrz1Ag40JqHvDUFfQcg+dQ3jaEOoau16y7EZGIe7oN8WYIXNMgWuMxH7r1QlanFbQZ+ziKjHmw==", "requires": { "@fluentui/set-version": "^8.2.0", "tslib": "^2.1.0" @@ -466,21 +464,21 @@ } }, "@fluentui/react": { - "version": "8.60.0", - "resolved": "https://registry.npmjs.org/@fluentui/react/-/react-8.60.0.tgz", - "integrity": "sha512-gMXIrCsRSaKG7WbJn1alme0D5epoWdM5QBuf/H6I63CoJHxscaqvejVk589a9k27zSb/KgtXnZO3sjUwdIFzgg==", - "requires": { - "@fluentui/date-time-utilities": "^8.4.0", - "@fluentui/font-icons-mdl2": "^8.2.1", - "@fluentui/foundation-legacy": "^8.2.1", - "@fluentui/merge-styles": "^8.4.0", - "@fluentui/react-focus": "^8.5.1", - "@fluentui/react-hooks": "^8.5.0", + "version": "8.64.1", + "resolved": "https://registry.npmjs.org/@fluentui/react/-/react-8.64.1.tgz", + "integrity": "sha512-2xk+LqUwn7Xp3eDIgeLInX4f6hgdie6RGcZo7D1nGSIB0p5EOR8IcwhataYs3HMte3WFWBRJhYbEXYZVfCfvfA==", + "requires": { + "@fluentui/date-time-utilities": "^8.5.0", + "@fluentui/font-icons-mdl2": "^8.2.5", + "@fluentui/foundation-legacy": "^8.2.5", + "@fluentui/merge-styles": "^8.5.0", + "@fluentui/react-focus": "^8.5.6", + "@fluentui/react-hooks": "^8.5.3", "@fluentui/react-window-provider": "^2.2.0", "@fluentui/set-version": "^8.2.0", - "@fluentui/style-utilities": "^8.6.1", - "@fluentui/theme": "^2.6.0", - "@fluentui/utilities": "^8.6.0", + "@fluentui/style-utilities": "^8.6.5", + "@fluentui/theme": "^2.6.4", + "@fluentui/utilities": "^8.8.1", "@microsoft/load-themed-styles": "^1.10.26", "tslib": "^2.1.0" }, @@ -493,15 +491,15 @@ } }, "@fluentui/react-focus": { - "version": "8.5.1", - "resolved": "https://registry.npmjs.org/@fluentui/react-focus/-/react-focus-8.5.1.tgz", - "integrity": "sha512-tqGf80l/YXU7EdyrQYC65b4pxtNmzw5CMCuvtLdc/t4oM3MDkSV4qfecGqv2dj1eyDcmmcL3djgUPzmbzCzXfw==", + "version": "8.5.6", + "resolved": "https://registry.npmjs.org/@fluentui/react-focus/-/react-focus-8.5.6.tgz", + "integrity": "sha512-K4INgXiwlinLz0Msa5TQbhL3YoN1mtqXO3p7XChDKeIf0QepAUDTnmJCrtKXqzY5VuXauLXyqm2darcHFrotVw==", "requires": { "@fluentui/keyboard-key": "^0.4.0", - "@fluentui/merge-styles": "^8.4.0", + "@fluentui/merge-styles": "^8.5.0", "@fluentui/set-version": "^8.2.0", - "@fluentui/style-utilities": "^8.6.1", - "@fluentui/utilities": "^8.6.0", + "@fluentui/style-utilities": "^8.6.5", + "@fluentui/utilities": "^8.8.1", "tslib": "^2.1.0" }, "dependencies": { @@ -513,13 +511,13 @@ } }, "@fluentui/react-hooks": { - "version": "8.5.0", - "resolved": "https://registry.npmjs.org/@fluentui/react-hooks/-/react-hooks-8.5.0.tgz", - "integrity": "sha512-RgcDI9mBnP3hjfXBQe3SWcPwLms74j/1t4LxxzIrK4LCXb3D4cMcUgoX0o68KIorGmihkmRPaJVBmbkMuy1VFA==", + "version": "8.5.3", + "resolved": "https://registry.npmjs.org/@fluentui/react-hooks/-/react-hooks-8.5.3.tgz", + "integrity": "sha512-zWiiEp/vYmKIXhw/bIVH9BnPKLT3zgWt9ryi3Tp1VVgxLEDloomM7RgDn0e2P8HZvynYRRT8GKq3XJ8H3AzAUg==", "requires": { "@fluentui/react-window-provider": "^2.2.0", "@fluentui/set-version": "^8.2.0", - "@fluentui/utilities": "^8.6.0", + "@fluentui/utilities": "^8.8.1", "tslib": "^2.1.0" }, "dependencies": { @@ -562,14 +560,14 @@ } }, "@fluentui/style-utilities": { - "version": "8.6.1", - "resolved": "https://registry.npmjs.org/@fluentui/style-utilities/-/style-utilities-8.6.1.tgz", - "integrity": "sha512-B1hrMZidn6DJGRTXDg9PLCcKNVMmC8EtDHRO/c6ZBvadZ21EwoSBEiR152w1/msA7nW1rlGxM1rb2k84gqKxww==", + "version": "8.6.5", + "resolved": "https://registry.npmjs.org/@fluentui/style-utilities/-/style-utilities-8.6.5.tgz", + "integrity": "sha512-X0ZIreRdUlnJnRQmKtngWEbrqFuecBlEv9fkKoHJeu+27AD+AU9RPYgK4fd/QCJRFoTU9WzQJj9tWylTBK+kFA==", "requires": { - "@fluentui/merge-styles": "^8.4.0", + "@fluentui/merge-styles": "^8.5.0", "@fluentui/set-version": "^8.2.0", - "@fluentui/theme": "^2.6.0", - "@fluentui/utilities": "^8.6.0", + "@fluentui/theme": "^2.6.4", + "@fluentui/utilities": "^8.8.1", "@microsoft/load-themed-styles": "^1.10.26", "tslib": "^2.1.0" }, @@ -582,13 +580,13 @@ } }, "@fluentui/theme": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@fluentui/theme/-/theme-2.6.0.tgz", - "integrity": "sha512-xLSb0LRhmq3RM3tstzuQrCso2kjolu7Nq2O0Oqn6r7ZD6VgXNZZLBLutHdzb11f+K8nIjHsFeqWXNnQx6ixQjA==", + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/@fluentui/theme/-/theme-2.6.4.tgz", + "integrity": "sha512-MXCY2Z5zc1nK5PCN6tqhI1nplX/LF+8lF8YsrSGvu6VDnHilxi4Rifv2dRc+jybNI13tZ50uteok2Je7jABcMA==", "requires": { - "@fluentui/merge-styles": "^8.4.0", + "@fluentui/merge-styles": "^8.5.0", "@fluentui/set-version": "^8.2.0", - "@fluentui/utilities": "^8.6.0", + "@fluentui/utilities": "^8.8.1", "tslib": "^2.1.0" }, "dependencies": { @@ -600,12 +598,12 @@ } }, "@fluentui/utilities": { - "version": "8.6.0", - "resolved": "https://registry.npmjs.org/@fluentui/utilities/-/utilities-8.6.0.tgz", - "integrity": "sha512-hnqIFtNaZkzhWFog7djEdHlIExCQ02QZo8f8UT5Z0glCVa9SaaOz4CFhQzKNS4RejAzKMSATckWXJlOxMIRKSg==", + "version": "8.8.1", + "resolved": "https://registry.npmjs.org/@fluentui/utilities/-/utilities-8.8.1.tgz", + "integrity": "sha512-JtICpdTINvifiyjp/LxxuDiU8BMkU73mzAP/3mz5KIhqPn3QjKG9ZeaBLPpoUXNOP/I6i9AYbAQHDvQI/ZK5tg==", "requires": { "@fluentui/dom-utilities": "^2.2.0", - "@fluentui/merge-styles": "^8.4.0", + "@fluentui/merge-styles": "^8.5.0", "@fluentui/set-version": "^8.2.0", "tslib": "^2.1.0" }, @@ -657,9 +655,9 @@ } }, "@microsoft/load-themed-styles": { - "version": "1.10.247", - "resolved": "https://registry.npmjs.org/@microsoft/load-themed-styles/-/load-themed-styles-1.10.247.tgz", - "integrity": "sha512-vKbuG3Mcbc4kkNAcIE13aIv5KoI2g+tHFFIZnFhtUilpYHc0VsMd4Fw7Jz81A8AB7L3wWu3OZB2CNiRnr1a3ew==" + "version": "1.10.252", + "resolved": "https://registry.npmjs.org/@microsoft/load-themed-styles/-/load-themed-styles-1.10.252.tgz", + "integrity": "sha512-l+vEBwEKrT+Bd2+q/jYtPfsBVn0iG7y28puPuppuPxPDW8fLQLy/W+/BJPK4GfCKbATIFXob7qK+g/dDnUjuRg==" }, "@nodelib/fs.scandir": { "version": "2.1.5", @@ -687,12 +685,6 @@ "fastq": "^1.6.0" } }, - "@types/cytoscape": { - "version": "3.19.4", - "resolved": "https://registry.npmjs.org/@types/cytoscape/-/cytoscape-3.19.4.tgz", - "integrity": "sha512-0IozTg1vdZrA3nuAK5o9Pa8nl2INUnTaXwcGwoiALDcsD8/TiVnp0Zi+R1IiPRG6edoy0Ya61/3osFLtfkhhmw==", - "dev": true - }, "@types/eslint": { "version": "8.4.1", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.4.1.tgz", @@ -735,9 +727,9 @@ } }, "@types/json-schema": { - "version": "7.0.9", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.9.tgz", - "integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==", + "version": "7.0.11", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz", + "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==", "dev": true }, "@types/node": { @@ -747,9 +739,9 @@ "dev": true }, "@types/office-js": { - "version": "1.0.240", - "resolved": "https://registry.npmjs.org/@types/office-js/-/office-js-1.0.240.tgz", - "integrity": "sha512-TYJy1tO3N4uTjlmNuUFh24DZFXodaGXISc1E0IWzeE93gj3xIcDlO3YgKGw/3zFBy1ZOEwViTSK1q2b5s6uX1g==", + "version": "1.0.246", + "resolved": "https://registry.npmjs.org/@types/office-js/-/office-js-1.0.246.tgz", + "integrity": "sha512-1/5tpaUj7DxIjdVhqBJ9JLICSgZc+HrvrvrKkEqjLD8+OxVB1uiZQfonoxoCEpxVdQwKVz9I6tdcWztz0JiXHA==", "dev": true }, "@types/office-runtime": { @@ -759,15 +751,15 @@ "dev": true }, "@types/prop-types": { - "version": "15.7.4", - "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.4.tgz", - "integrity": "sha512-rZ5drC/jWjrArrS8BR6SIr4cWpW09RNTYt9AMZo3Jwwif+iacXAqgVjm0B0Bv/S1jhDXKHqRVNCbACkJ89RAnQ==", + "version": "15.7.5", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", + "integrity": "sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==", "dev": true }, "@types/react": { - "version": "16.14.23", - "resolved": "https://registry.npmjs.org/@types/react/-/react-16.14.23.tgz", - "integrity": "sha512-WngBZLuSkP4IAgPi0HOsGCHo6dn3CcuLQnCfC17VbA7YBgipZiZoTOhObwl/93DsFW0Y2a/ZXeonpW4DxirEJg==", + "version": "16.14.24", + "resolved": "https://registry.npmjs.org/@types/react/-/react-16.14.24.tgz", + "integrity": "sha512-e7U2WC8XQP/xfR7bwhOhNFZKPTfW1ph+MiqtudKb8tSV8RyCsovQx2sNVtKoOryjxFKpHPPC/yNiGfdeVM5Gyw==", "dev": true, "requires": { "@types/prop-types": "*", @@ -775,16 +767,6 @@ "csstype": "^3.0.2" } }, - "@types/react-cytoscapejs": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@types/react-cytoscapejs/-/react-cytoscapejs-1.2.2.tgz", - "integrity": "sha512-Z+CeQLw4XyQOrS/SdbcUCdYfWqYmaPS/WT+gNy33KYR09BtVzekx1jeNU5kNDvXCv8/8jGlhSrSk0uO7ocGuFg==", - "dev": true, - "requires": { - "@types/cytoscape": "*", - "@types/react": "*" - } - }, "@types/react-dom": { "version": "16.9.14", "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-16.9.14.tgz", @@ -819,9 +801,9 @@ "dev": true }, "@types/uglify-js": { - "version": "3.13.1", - "resolved": "https://registry.npmjs.org/@types/uglify-js/-/uglify-js-3.13.1.tgz", - "integrity": "sha512-O3MmRAk6ZuAKa9CHgg0Pr0+lUOqoMLpc9AS4R8ano2auvsg7IE8syF3Xh/NPr26TWklxYcqoEEFdzLLs1fV9PQ==", + "version": "3.13.2", + "resolved": "https://registry.npmjs.org/@types/uglify-js/-/uglify-js-3.13.2.tgz", + "integrity": "sha512-/xFrPIo+4zOeNGtVMbf9rUm0N+i4pDf1ynExomqtokIJmVzR3962lJ1UE+MmexMkA0cmN9oTzg5Xcbwge0Ij2Q==", "dev": true, "requires": { "source-map": "^0.6.1" @@ -893,12 +875,12 @@ }, "dependencies": { "semver": { - "version": "7.3.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", - "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "version": "7.3.6", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.6.tgz", + "integrity": "sha512-HZWqcgwLsjaX1HBD31msI/rXktuIhS+lWvdE4kN9z+8IVT4Itc7vqU2WvYsyD6/sjYCt4dEKH/m1M3dwI9CC5w==", "dev": true, "requires": { - "lru-cache": "^6.0.0" + "lru-cache": "^7.4.0" } } } @@ -972,12 +954,12 @@ }, "dependencies": { "semver": { - "version": "7.3.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", - "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "version": "7.3.6", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.6.tgz", + "integrity": "sha512-HZWqcgwLsjaX1HBD31msI/rXktuIhS+lWvdE4kN9z+8IVT4Itc7vqU2WvYsyD6/sjYCt4dEKH/m1M3dwI9CC5w==", "dev": true, "requires": { - "lru-cache": "^6.0.0" + "lru-cache": "^7.4.0" } } } @@ -1393,15 +1375,28 @@ "dev": true }, "babel-loader": { - "version": "8.2.3", - "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-8.2.3.tgz", - "integrity": "sha512-n4Zeta8NC3QAsuyiizu0GkmRcQ6clkV9WFUnUf1iXP//IeSKbWjofW3UHyZVwlOB4y039YQKefawyTn64Zwbuw==", + "version": "8.2.4", + "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-8.2.4.tgz", + "integrity": "sha512-8dytA3gcvPPPv4Grjhnt8b5IIiTcq/zeXOPk4iTYI0SVXcsmuGg7JtBRDp8S9X+gJfhQ8ektjXZlDu1Bb33U8A==", "dev": true, "requires": { "find-cache-dir": "^3.3.1", - "loader-utils": "^1.4.0", + "loader-utils": "^2.0.0", "make-dir": "^3.1.0", "schema-utils": "^2.6.5" + }, + "dependencies": { + "loader-utils": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.2.tgz", + "integrity": "sha512-TM57VeHptv569d/GKh6TAYdzKblwDNiumOdkFnejjD0XwTH87K90w3O7AiJRqdQoXygvi1VQTJTLGhJl7WqA7A==", + "dev": true, + "requires": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^2.1.2" + } + } } }, "balanced-match": { @@ -1422,6 +1417,11 @@ "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==", "dev": true }, + "bigdecimal": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/bigdecimal/-/bigdecimal-0.6.1.tgz", + "integrity": "sha512-pFxbEwaZRK6XifePcw22DxJkCzCMw3YAossM884Tkvh9QpdWzFBLkgdnjx29yEickWBVYv1F9DdaDxDaF3dzkw==" + }, "binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -1511,13 +1511,13 @@ } }, "browserslist": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.20.0.tgz", - "integrity": "sha512-bnpOoa+DownbciXj0jVGENf8VYQnE2LNWomhYuCsMmmx9Jd9lwq0WXODuwpSsp8AVdKM2/HorrzxAfbKvWTByQ==", + "version": "4.20.2", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.20.2.tgz", + "integrity": "sha512-CQOBCqp/9pDvDbx3xfMi+86pr4KXIf2FDkTTdeuYw8OxS9t898LA1Khq57gtufFILXpfgsSx5woNgsBgvGjpsA==", "dev": true, "requires": { - "caniuse-lite": "^1.0.30001313", - "electron-to-chromium": "^1.4.76", + "caniuse-lite": "^1.0.30001317", + "electron-to-chromium": "^1.4.84", "escalade": "^3.1.1", "node-releases": "^2.0.2", "picocolors": "^1.0.0" @@ -1576,9 +1576,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001314", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001314.tgz", - "integrity": "sha512-0zaSO+TnCHtHJIbpLroX7nsD+vYuOVjl3uzFbJO1wMVbuveJA0RK2WcQA9ZUIOiO0/ArMiMgHJLxfEZhQiC0kw==", + "version": "1.0.30001327", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001327.tgz", + "integrity": "sha512-1/Cg4jlD9qjZzhbzkzEaAC2JHsP0WrOc8Rd/3a3LuajGzGWR/hD7TVyvq99VqmTy99eVh8Zkmdq213OgvgXx7w==", "dev": true }, "chalk": { @@ -1909,22 +1909,22 @@ } }, "css-select": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.2.1.tgz", - "integrity": "sha512-/aUslKhzkTNCQUB2qTX84lVmfia9NyjP3WpDGtj/WxhwBzWBYUV3DgUpurHTme8UTPcPlAD1DJ+b0nN/t50zDQ==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.3.0.tgz", + "integrity": "sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ==", "dev": true, "requires": { "boolbase": "^1.0.0", - "css-what": "^5.1.0", - "domhandler": "^4.3.0", + "css-what": "^6.0.1", + "domhandler": "^4.3.1", "domutils": "^2.8.0", "nth-check": "^2.0.1" } }, "css-what": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/css-what/-/css-what-5.1.0.tgz", - "integrity": "sha512-arSMRWIIFY0hV8pIxZMEfmMI47Wj3R/aWpZDDxWYCPEiOMv6tfOrnpDtgxBYPEQD4V0Y/958+1TdC3iWTFcUPw==", + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", "dev": true }, "csstype": { @@ -1933,27 +1933,6 @@ "integrity": "sha512-sa6P2wJ+CAbgyy4KFssIb/JNMLxFvKF1pCYCSXS8ZMuqZnMsrxqI2E5sPyoTpxoPU/gVZMzr2zjOfg8GIZOMsw==", "dev": true }, - "cytoscape": { - "version": "3.21.0", - "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.21.0.tgz", - "integrity": "sha512-xPINMzQNGN4WIog93gYRScT4y/CyZMmqunnhU59/8LynhWwzepJtUydMfvIPuz5TmJ9rSCMdw6rmxhfbb1eofw==", - "dev": true, - "requires": { - "heap": "^0.2.6", - "lodash.debounce": "^4.0.8", - "lodash.get": "^4.4.2", - "lodash.set": "^4.3.2", - "lodash.topath": "^4.5.2" - } - }, - "cytoscape-klay": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/cytoscape-klay/-/cytoscape-klay-3.1.4.tgz", - "integrity": "sha512-VwPj0VR25GPfy6qXVQRi/MYlZM/zkdvRhHlgqbM//lSvstgM6fhp3ik/uM8Wr8nlhskfqz/M1fIDmR6UckbS2A==", - "requires": { - "klayjs": "^0.4.1" - } - }, "debug": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", @@ -2118,15 +2097,15 @@ "dev": true }, "domelementtype": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.2.0.tgz", - "integrity": "sha512-DtBMo82pv1dFtUmHyr48beiuq792Sxohr+8Hm9zoxklYPfa6n0Z3Byjj2IV7bmr2IyqClnqEQhfgHJJ5QF0R5A==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", "dev": true }, "domhandler": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.3.0.tgz", - "integrity": "sha512-fC0aXNQXqKSFTr2wDNZDhsEYjCiYsDWl3D01kwt25hm1YIPyDGHvvi3rw+PLqHAl/m71MaiF7d5zvBr0p5UB2g==", + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.3.1.tgz", + "integrity": "sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==", "dev": true, "requires": { "domelementtype": "^2.2.0" @@ -2168,9 +2147,9 @@ "dev": true }, "electron-to-chromium": { - "version": "1.4.78", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.78.tgz", - "integrity": "sha512-o61+D/Lx7j/E0LIin/efOqeHpXhwi1TaQco9vUcRmr91m25SfZY6L5hWJDv/r+6kNjboFKgBw1LbfM0lbhuK6Q==", + "version": "1.4.106", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.106.tgz", + "integrity": "sha512-ZYfpVLULm67K7CaaGP7DmjyeMY4naxsbTy+syVVxT6QHI1Ww8XbJjmr9fDckrhq44WzCrcC5kH3zGpdusxwwqg==", "dev": true }, "emitter-listener": { @@ -2242,9 +2221,9 @@ } }, "es-abstract": { - "version": "1.19.1", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.19.1.tgz", - "integrity": "sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w==", + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.19.2.tgz", + "integrity": "sha512-gfSBJoZdlL2xRiOCy0g8gLMryhoe1TlimjzU99L/31Z8QEGIhVQI+EWwt5lT+AuU9SnorVupXFqqOGqGfsyO6w==", "dev": true, "requires": { "call-bind": "^1.0.2", @@ -2253,15 +2232,15 @@ "get-intrinsic": "^1.1.1", "get-symbol-description": "^1.0.0", "has": "^1.0.3", - "has-symbols": "^1.0.2", + "has-symbols": "^1.0.3", "internal-slot": "^1.0.3", "is-callable": "^1.2.4", - "is-negative-zero": "^2.0.1", + "is-negative-zero": "^2.0.2", "is-regex": "^1.1.4", "is-shared-array-buffer": "^1.0.1", "is-string": "^1.0.7", - "is-weakref": "^1.0.1", - "object-inspect": "^1.11.0", + "is-weakref": "^1.0.2", + "object-inspect": "^1.12.0", "object-keys": "^1.1.1", "object.assign": "^4.1.2", "string.prototype.trimend": "^1.0.4", @@ -2269,6 +2248,12 @@ "unbox-primitive": "^1.0.1" }, "dependencies": { + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true + }, "is-regex": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", @@ -2419,9 +2404,9 @@ "dev": true }, "globals": { - "version": "13.12.1", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.12.1.tgz", - "integrity": "sha512-317dFlgY2pdJZ9rspXDks7073GpDmXdfbM3vYYp0HAMKGDh1FfWPleI2ljVNLQX5M5lXcAslTcPTrOrMEFOjyw==", + "version": "13.13.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.13.0.tgz", + "integrity": "sha512-EQ7Q18AJlPwp3vUDL4mKA0KXrXyNIQyWon6T6XQiBQF0XHvRsiCSrWmmeATpUzdJN2HhWZU6Pdl0a9zdep5p6A==", "dev": true, "requires": { "type-fest": "^0.20.2" @@ -2440,12 +2425,12 @@ "dev": true }, "semver": { - "version": "7.3.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", - "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "version": "7.3.6", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.6.tgz", + "integrity": "sha512-HZWqcgwLsjaX1HBD31msI/rXktuIhS+lWvdE4kN9z+8IVT4Itc7vqU2WvYsyD6/sjYCt4dEKH/m1M3dwI9CC5w==", "dev": true, "requires": { - "lru-cache": "^6.0.0" + "lru-cache": "^7.4.0" } }, "supports-color": { @@ -2495,9 +2480,9 @@ } }, "eslint-plugin-react": { - "version": "7.29.3", - "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.29.3.tgz", - "integrity": "sha512-MzW6TuCnDOcta67CkpDyRfRsEVx9FNMDV8wZsDqe1luHPdGTrQIUaUXD27Ja3gHsdOIs/cXzNchWGlqm+qRVRg==", + "version": "7.29.4", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.29.4.tgz", + "integrity": "sha512-CVCXajliVh509PcZYRFyu/BoUEz452+jtQJq2b3Bae4v3xBUWPLCmtmBM+ZinG4MzwmxJgJ2M5rMqhqLVn7MtQ==", "dev": true, "requires": { "array-includes": "^3.1.4", @@ -3229,13 +3214,13 @@ "dev": true }, "micromatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.4.tgz", - "integrity": "sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", + "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", "dev": true, "requires": { - "braces": "^3.0.1", - "picomatch": "^2.2.3" + "braces": "^3.0.2", + "picomatch": "^2.3.1" } }, "picomatch": { @@ -3300,12 +3285,6 @@ "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", "dev": true }, - "heap": { - "version": "0.2.7", - "resolved": "https://registry.npmjs.org/heap/-/heap-0.2.7.tgz", - "integrity": "sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==", - "dev": true - }, "hoist-non-react-statics": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", @@ -3380,9 +3359,9 @@ }, "dependencies": { "clean-css": { - "version": "5.2.4", - "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-5.2.4.tgz", - "integrity": "sha512-nKseG8wCzEuji/4yrgM/5cthL9oTDc5UOQyFMvW/Q53oP6gLH690o1NbuTh6Y18nujr7BxlsFuS7gXLnLzKJGg==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-5.3.0.tgz", + "integrity": "sha512-YYuuxv4H/iNb1Z/5IbMRoxgrzjWGhOEFfd+groZ5dMCVkpENiMZmwspdrzBo9286JjM1gZJPAyL7ZIdzuvu2AQ==", "dev": true, "requires": { "source-map": "~0.6.0" @@ -3416,9 +3395,9 @@ "dev": true }, "terser": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/terser/-/terser-5.12.0.tgz", - "integrity": "sha512-R3AUhNBGWiFc77HXag+1fXpAxTAFRQTJemlJKjAgD9r8xXTpjNKqIXwHM/o7Rh+O0kUJtS3WQVdBeMKFk5sw9A==", + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.12.1.tgz", + "integrity": "sha512-NXbs+7nisos5E+yXwAD+y7zrcTkMqb0dEJxIGtSKPdCBzopf7ni4odPul2aechpV7EXNvOudYOX2bb5tln1jbQ==", "dev": true, "requires": { "acorn": "^8.5.0", @@ -3808,9 +3787,9 @@ "dev": true }, "is-number-object": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.6.tgz", - "integrity": "sha512-bEVOqiRcvo3zO1+G2lVMy+gkkEm9Yh7cDMRusKKu5ZJKPUYSJwICTKZrNKHA2EbSP0Tu0+6B/emsYNHZyn6K8g==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz", + "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", "dev": true, "requires": { "has-tostringtag": "^1.0.0" @@ -3848,10 +3827,13 @@ } }, "is-shared-array-buffer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.1.tgz", - "integrity": "sha512-IU0NmyknYZN0rChcKhRO1X8LYz5Isj/Fsqh8NJOSf+N/hCOTwy29F32Ik7a+QszE63IdvmwdTPDd6cZ5pg4cwA==", - "dev": true + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", + "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==", + "dev": true, + "requires": { + "call-bind": "^1.0.2" + } }, "is-stream": { "version": "2.0.1", @@ -3944,6 +3926,11 @@ } } }, + "js-lexer": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/js-lexer/-/js-lexer-0.1.2.tgz", + "integrity": "sha512-ww9L8rZggh1gJS3zkM0uFh6OfjEamei28BY3hpcepDImx/wWcNk/jzsDPGzVbdiAiokd41YINZaiFj0PrWd8Vg==" + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -4008,19 +3995,19 @@ } }, "jsx-ast-utils": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.2.1.tgz", - "integrity": "sha512-uP5vu8xfy2F9A6LGC22KO7e2/vGTS1MhP+18f++ZNlf0Ohaxbc9nIEwHAsejlJKyzfZzU5UIhe5ItYkitcZnZA==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.2.2.tgz", + "integrity": "sha512-HDAyJ4MNQBboGpUnHAVUNJs6X0lh058s6FuixsFGP7MgJYpD6Vasd6nzSG5iIfXu1zAYlHJ/zsOKNlrenTUBnw==", "dev": true, "requires": { - "array-includes": "^3.1.3", + "array-includes": "^3.1.4", "object.assign": "^4.1.2" } }, "jszip": { - "version": "3.7.1", - "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.7.1.tgz", - "integrity": "sha512-ghL0tz1XG9ZEmRMcEN2vt7xabrDdqHHeykgARpmZ0BiIctWxM47Vt63ZO2dnp4QYt/xJVLLy5Zv1l/xRdh2byg==", + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.9.1.tgz", + "integrity": "sha512-H9A60xPqJ1CuC4Ka6qxzXZeU8aNmgOeP5IFqwJbQQwtu2EUYxota3LdsiZWplF7Wgd9tkAd0mdu36nceSaPuYw==", "dev": true, "requires": { "lie": "~3.3.0", @@ -4041,11 +4028,6 @@ "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true }, - "klayjs": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/klayjs/-/klayjs-0.4.1.tgz", - "integrity": "sha1-W/n63Ho+RLlAgrulAefYAwdtz8I=" - }, "klona": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/klona/-/klona-2.0.5.tgz", @@ -4167,42 +4149,18 @@ "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "dev": true }, - "lodash.debounce": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", - "integrity": "sha1-gteb/zCmfEAF/9XiUVMArZyk168=", - "dev": true - }, - "lodash.get": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", - "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=", - "dev": true - }, "lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, - "lodash.set": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", - "integrity": "sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM=", - "dev": true - }, "lodash.sortby": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=", "dev": true }, - "lodash.topath": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/lodash.topath/-/lodash.topath-4.5.2.tgz", - "integrity": "sha1-NhY1Hzu6YZlKCTGYlmC9AyVP0Ak=", - "dev": true - }, "lodash.truncate": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", @@ -4235,13 +4193,10 @@ } }, "lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, - "requires": { - "yallist": "^4.0.0" - } + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.8.0.tgz", + "integrity": "sha512-AmXqneQZL3KZMIgBpaPTeI6pfwh+xQ2vutMsyqOu1TBdEXFZgpG/80wuJ531w2ZN7TI0/oc8CPxzh/DKQudZqg==", + "dev": true }, "make-dir": { "version": "3.1.0", @@ -4376,9 +4331,9 @@ "dev": true }, "node-forge": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.2.1.tgz", - "integrity": "sha512-Fcvtbb+zBcZXbTTVwqGA5W+MKBj56UjVRevvchv5XrcyXbmNdesfZL37nlcWOfpgHhgmxApw3tQbTr4CqNmX4w==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", + "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==", "dev": true } } @@ -5147,9 +5102,9 @@ "dev": true }, "prettier": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.5.1.tgz", - "integrity": "sha512-vBZcPRUR5MZJwoyi3ZoyQlc1rXeEck8KgeC9AwwOn+exuxLxq5toTRDTSaVrXHxelDMHy9zlicw8u66yxoSUFg==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.6.2.tgz", + "integrity": "sha512-PkUpF+qoXTqhOeWL9fu7As8LXsIUZ1WYaJiY/a7McAQzxjk82OF0tibkFXVCDImZtWxbvojFjerkiLb0/q8mew==", "dev": true }, "prettier-linter-helpers": { @@ -5277,16 +5232,6 @@ "prop-types": "^15.6.2" } }, - "react-cytoscapejs": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/react-cytoscapejs/-/react-cytoscapejs-1.2.1.tgz", - "integrity": "sha512-8exVCetpzyGCAKuRjXPWGjFCnb22boZ3SXUPpPB/+wQI8Q8BwkT1URN3A7J1Czvj1qAbShh5QQ514mBUp7i7kw==", - "dev": true, - "requires": { - "cytoscape": "^3.2.19", - "prop-types": "^15.6.2" - } - }, "react-dom": { "version": "16.14.0", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-16.14.0.tgz", @@ -5972,19 +5917,37 @@ } }, "string.prototype.matchall": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.6.tgz", - "integrity": "sha512-6WgDX8HmQqvEd7J+G6VtAahhsQIssiZ8zl7zKh1VDMFyL3hRTJP4FTNA3RbIp2TOQ9AYNDcc7e3fH0Qbup+DBg==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.7.tgz", + "integrity": "sha512-f48okCX7JiwVi1NXCVWcFnZgADDC/n2vePlQ/KUCNqCikLLilQvwjMO8+BHVKvgzH0JB0J9LEPgxOGT02RoETg==", "dev": true, "requires": { "call-bind": "^1.0.2", "define-properties": "^1.1.3", "es-abstract": "^1.19.1", "get-intrinsic": "^1.1.1", - "has-symbols": "^1.0.2", + "has-symbols": "^1.0.3", "internal-slot": "^1.0.3", - "regexp.prototype.flags": "^1.3.1", + "regexp.prototype.flags": "^1.4.1", "side-channel": "^1.0.4" + }, + "dependencies": { + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true + }, + "regexp.prototype.flags": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.1.tgz", + "integrity": "sha512-pMR7hBVUUGI7PMA37m2ofIdQCsomVnas+Jn5UPGAHQ+/LlwKm/aTLJHdasmHRzlfeZwHiAOaRSo2rbBDm3nNUQ==", + "dev": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + } + } } }, "string.prototype.trimend": { @@ -6066,9 +6029,9 @@ }, "dependencies": { "ajv": { - "version": "8.10.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.10.0.tgz", - "integrity": "sha512-bzqAEZOjkrUMl2afH8dknrq5KEk2SrwdBROR+vH1EKVQTqaUbJVPdc/gEdggTMM0Se+s+Ja4ju4TlNcStKl2Hw==", + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", + "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==", "dev": true, "requires": { "fast-deep-equal": "^3.1.1", @@ -6153,9 +6116,9 @@ "dev": true }, "terser": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/terser/-/terser-5.12.0.tgz", - "integrity": "sha512-R3AUhNBGWiFc77HXag+1fXpAxTAFRQTJemlJKjAgD9r8xXTpjNKqIXwHM/o7Rh+O0kUJtS3WQVdBeMKFk5sw9A==", + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.12.1.tgz", + "integrity": "sha512-NXbs+7nisos5E+yXwAD+y7zrcTkMqb0dEJxIGtSKPdCBzopf7ni4odPul2aechpV7EXNvOudYOX2bb5tln1jbQ==", "dev": true, "requires": { "acorn": "^8.5.0", @@ -6229,9 +6192,9 @@ "dev": true }, "ts-loader": { - "version": "9.2.7", - "resolved": "https://registry.npmjs.org/ts-loader/-/ts-loader-9.2.7.tgz", - "integrity": "sha512-Fxh44mKli9QezgbdCXkEJWxnedQ0ead7DXTH+lfXEPedu+Y9EtMJ2aQ9G3Dj1j7Q612E8931rww8NDZha4Tibg==", + "version": "9.2.8", + "resolved": "https://registry.npmjs.org/ts-loader/-/ts-loader-9.2.8.tgz", + "integrity": "sha512-gxSak7IHUuRtwKf3FIPSW1VpZcqF9+MBrHOvBp9cjHh+525SjtCIJKVGjRKIAfxBwDGDGCFF00rTfzB1quxdSw==", "dev": true, "requires": { "chalk": "^4.1.0", @@ -6281,12 +6244,12 @@ "dev": true }, "semver": { - "version": "7.3.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", - "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "version": "7.3.6", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.6.tgz", + "integrity": "sha512-HZWqcgwLsjaX1HBD31msI/rXktuIhS+lWvdE4kN9z+8IVT4Itc7vqU2WvYsyD6/sjYCt4dEKH/m1M3dwI9CC5w==", "dev": true, "requires": { - "lru-cache": "^6.0.0" + "lru-cache": "^7.4.0" } }, "supports-color": { @@ -6341,9 +6304,9 @@ } }, "typescript": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.6.2.tgz", - "integrity": "sha512-HM/hFigTBHZhLXshn9sN37H085+hQGeJHJ/X7LpBWLID/fbc2acUMfU+lGD98X81sKP+pFa9f0DZmCwB9GnbAg==", + "version": "4.6.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.6.3.tgz", + "integrity": "sha512-yNIatDa5iaofVozS/uQJEl3JRWLKKGJKh6Yaiv0GLGSuhpFJe7P3SbHZ8/yjAHRQwKRoA6YZqlfjXWmVzoVSMw==", "dev": true }, "unbox-primitive": { @@ -6476,9 +6439,9 @@ "dev": true }, "webpack": { - "version": "5.70.0", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.70.0.tgz", - "integrity": "sha512-ZMWWy8CeuTTjCxbeaQI21xSswseF2oNOwc70QSKNePvmxE7XW36i7vpBMYZFAUHPwQiEbNGCEYIOOlyRbdGmxw==", + "version": "5.72.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.72.0.tgz", + "integrity": "sha512-qmSmbspI0Qo5ld49htys8GY9XhS9CGqFoHTsOVAnjBdg0Zn79y135R+k4IR4rKK6+eKaabMhJwiVB7xw0SJu5w==", "dev": true, "requires": { "@types/eslint-scope": "^3.7.3", @@ -6508,9 +6471,9 @@ }, "dependencies": { "graceful-fs": { - "version": "4.2.9", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz", - "integrity": "sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ==", + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", "dev": true }, "schema-utils": { @@ -6874,12 +6837,6 @@ "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", "dev": true - }, - "yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true } } } diff --git a/add-in/package.json b/add-in/package.json index 87f92da..0f95a15 100644 --- a/add-in/package.json +++ b/add-in/package.json @@ -30,9 +30,11 @@ "dependencies": { "@fluentui/font-icons-mdl2": "^8.1.24", "@fluentui/react": "^8.55.3", + "bigdecimal": "^0.6.1", "core-js": "^3.21.1", "cytoscape-klay": "^3.1.4", "es6-promise": "^4.2.8", + "js-lexer": "^0.1.2", "react": "^16.14.0", "react-dom": "^16.14.0", "regenerator-runtime": "^0.13.9" diff --git a/add-in/src/api/pids/archive/lexer.ts b/add-in/src/api/pids/archive/lexer.ts new file mode 100644 index 0000000..6e7a00b --- /dev/null +++ b/add-in/src/api/pids/archive/lexer.ts @@ -0,0 +1,320 @@ +// /** This character denotes the end of file */ +// const YYEOF = -1; + +// /** initial size of the lookahead buffer */ +// const ZZ_BUFFERSIZE = 16384; + +// /** lexical states */ +// const YYINITIAL = 0; + +// /** +// * ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l +// * ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l +// * at the beginning of a line +// * l is of the form l = 2*k, k a non negative integer +// */ +// const ZZ_LEXSTATE = [0, 0]; + +// /** +// * Translates characters to character classes +// */ +// const ZZ_CMAP_PACKED = +// "\11\0\1\1\1\1\1\0\1\1\1\1\22\0\1\1\7\0\1\0" + +// "\1\0\2\0\1\0\1\0\1\0\1\0\12\2\1\0\1\0\5\0" + +// "\32\3\1\0\1\0\1\0\1\0\1\0\1\0\32\3\1\0\1\0" + +// "\1\0\uff82\0"; + +// const ZZ_ACTION_PACKED_0 = "\1\0\1\1\1\2\1\3\1\4"; + +// const ZZ_ROWMAP_PACKED_0 = "\0\0\0\4\0\10\0\14\0\20"; + +// const ZZ_TRANS_PACKED_0 = "\1\2\1\3\1\4\1\5\5\0\1\3\4\0\1\4\4\0\1\5"; + +// const ZZ_ATTRIBUTE_PACKED_0 = "\1\0\1\11\3\1"; + +// /** +// * Translates characters to character classes +// */ +// const ZZ_CMAP = zzUnpackCMap(ZZ_CMAP_PACKED); + +// /** +// * Translates DFA states to action switch labels. +// */ +// const ZZ_ACTION = zzUnpackAction(); + +// function zzUnpackAction() { +// let result = Array(5); +// let offset = 0; +// offset = zzUnpackActionArgs(ZZ_ACTION_PACKED_0, offset, result); +// return result; +// } + +// function zzUnpackActionArgs(packed, offset, result) { +// let i = 0; /* index in packed string */ +// let j = offset; /* index in unpacked array */ +// let l = packed.length; +// while (i < l) { +// let count = packed.charAt(i++); +// let value = packed.charAt(i++); +// do { +// result[j++] = value; +// } while (--count > 0); +// } +// return j; +// } + +// /** +// * Translates a state to a row index in the transition table +// */ +// const ZZ_ROWMAP = zzUnpackRowMap(); + +// function zzUnpackRowMap() { +// let result = Array(5); +// let offset = 0; +// offset = zzUnpackRowMapArgs(ZZ_ROWMAP_PACKED_0, offset, result); +// return result; +// } + +// function zzUnpackRowMapArgs(packed, offset, result) { +// let i = 0; /* index in packed string */ +// let j = offset; /* index in unpacked array */ +// let l = packed.length; +// while (i < l) { +// let high = packed.charAt(i++) << 16; +// result[j++] = high | packed.charAt(i++); +// } +// return j; +// } + +// /** +// * The transition table of the DFA +// */ +// const ZZ_TRANS = zzUnpackTrans(); + +// function zzUnpackTrans() { +// let result = new Array(20); +// let offset = 0; +// offset = zzUnpackTransArgs(ZZ_TRANS_PACKED_0, offset, result); +// return result; +// } + +// function zzUnpackTransArgs(packed, offset, result) { +// let i = 0; /* index in packed string */ +// let j = offset; /* index in unpacked array */ +// let l = packed.length; +// while (i < l) { +// let count = packed.charAt(i++); +// let value = packed.charAt(i++); +// value--; +// do result[j++] = value; +// while (--count > 0); +// } +// return j; +// } + +// /* error codes */ +// const ZZ_UNKNOWN_ERROR = 0; +// const ZZ_NO_MATCH = 1; +// const ZZ_PUSHBACK_2BIG = 2; + +// /* error messages for the codes above */ +// const ZZ_ERROR_MSG = [ +// "Unkown internal scanner error", +// "Error: could not match input", +// "Error: pushback value was too large", +// ]; + +// /** +// * ZZ_ATTRIBUTE[aState] contains the attributes of state aState +// */ +// const ZZ_ATTRIBUTE = zzUnpackAttribute(); + +// function zzUnpackAttribute() { +// let result = Array(5); +// let offset = 0; +// offset = zzUnpackAttributeArgs(ZZ_ATTRIBUTE_PACKED_0, offset, result); +// return result; +// } + +// function zzUnpackAttributeArgs(packed, offset, result) { +// let i = 0; /* index in packed string */ +// let j = offset; /* index in unpacked array */ +// let l = packed.length; +// while (i < l) { +// let count = packed.charAt(i++); +// let value = packed.charAt(i++); +// do { +// result[j++] = value; +// } while (--count > 0); +// } +// return j; +// } + +// /** +// * Unpacks the compressed character translation table. +// * +// * @param packed the packed character translation table +// * @return the unpacked character translation table +// */ +// function zzUnpackCMap(packed) { +// let map = Array(0x10000); +// let i = 0; /* index in packed string */ +// let j = 0; /* index in unpacked array */ +// while (i < 64) { +// let count = packed.charAt(i++); +// let value = packed.charAt(i++); +// do { +// map[j++] = value; +// } while (--count > 0); +// } +// return map; +// } + +// class LexerOld { +// /** the input device */ +// zzReader; + +// /** the current state of the DFA */ +// zzState; + +// /** the current lexical state */ +// zzLexicalState = YYINITIAL; + +// /** +// * this buffer contains the current text to be matched and is +// * the source of the yytext() string +// */ +// zzBuffer = Array(ZZ_BUFFERSIZE); + +// /** the textposition at the last accepting state */ +// zzMarkedPos; + +// /** the current text position in the buffer */ +// zzCurrentPos; + +// /** startRead marks the beginning of the yytext() string in the buffer */ +// zzStartRead; + +// /** +// * endRead marks the last character in the buffer, that has been read +// * from input +// */ +// zzEndRead; + +// /** number of newlines encountered up to the start of the matched text */ +// yyline; + +// /** the number of characters up to the start of the matched text */ +// yychar; + +// /** +// * the number of characters from the last newline up to the start of the +// * matched text +// */ +// yycolumn; + +// /** +// * zzAtBOL == true <=> the scanner is currently at the beginning of a line +// */ +// zzAtBOL = true; + +// /** zzAtEOF == true <=> the scanner is at the EOF */ +// zzAtEOF; + +// /** denotes if the user-EOF-code has already been executed */ +// zzEOFDone; + +// constructor(reader) { +// this.zzReader = reader; +// } + +// scan() { +// let zzInput; +// let zzAction; + +// // cached fields: +// let zzCurrentPosL; +// let zzMarkedPosL; +// let zzEndReadL = this.zzEndRead; +// let zzBufferL = this.zzBuffer; +// let zzCMapL = ZZ_CMAP; + +// let zzTransL = ZZ_TRANS; +// let zzRowMapL = ZZ_ROWMAP; +// let zzAttrL = ZZ_ATTRIBUTE; + +// zzForAction: { +// while (true) { +// if (zzCurrentPosL < zzEndReadL) { +// zzInput = zzBufferL[zzCurrentPosL++]; +// } else if (this.zzAtEOF) { +// zzInput = YYEOF; +// break zzForAction; +// } else { +// // store back cached positions +// this.zzCurrentPos = zzCurrentPosL; +// this.zzMarkedPos = zzMarkedPosL; +// let eof = ""; +// // get translated positions and possibly new buffer +// zzCurrentPosL = this.zzCurrentPos; +// zzMarkedPosL = this.zzMarkedPos; +// zzBufferL = this.zzBuffer; +// zzEndReadL = this.zzEndRead; +// if (eof) { +// zzInput = YYEOF; +// break zzForAction; +// } else { +// zzInput = zzBufferL[zzCurrentPosL++]; +// } +// } +// let zzNext = zzTransL[zzRowMapL[this.zzState] + zzCMapL[zzInput]]; +// if (zzNext == -1) { +// break zzForAction; +// } +// this.zzState = zzNext; + +// let zzAttributes = zzAttrL[this.zzState]; +// if ((zzAttributes & 1) == 1) { +// zzAction = this.zzState; +// zzMarkedPosL = zzCurrentPosL; +// if ((zzAttributes & 8) == 8) { +// break zzForAction; +// } +// } + +// // store back cached position +// this.zzMarkedPos = zzMarkedPosL; + +// switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) { +// case 2: { +// return new TSpace(); +// } +// case 5: +// break; +// case 1: { +// return new TSymbol(yytext()); +// } +// case 6: +// break; +// case 4: { +// return new TWord(yytext()); +// } +// case 7: +// break; +// case 3: { +// return new TInt(yytext()); +// } +// case 8: +// break; +// default: +// if (zzInput == YYEOF && this.zzStartRead == this.zzCurrentPos) { +// this.zzAtEOF = true; +// return null; +// } else { +// zzScanError(ZZ_NO_MATCH); +// } +// } +// } +// } +// } +// } diff --git a/add-in/src/api/pids/archive/parsingTest.ts b/add-in/src/api/pids/archive/parsingTest.ts new file mode 100644 index 0000000..1f7b46b --- /dev/null +++ b/add-in/src/api/pids/archive/parsingTest.ts @@ -0,0 +1,90 @@ + +const pattern = "(`(- )``;`=`IF`(``>` ``,``,` ``)`;`=`SUM`(``,` `B``)`;`=`C``+`)" +const ori_pat = "(,(-,),,;,=,IF,(,,>, ,,,,,,, ,,),;,=,SUM,(,,,, ,B,,),;,=,C,,+,)" + +const formMatrix = [ + ["-5", "=IF(A16 > 0, A16, 0)", "=SUM(A16, B16)", "=C16+1"], + ["-4", "=IF(A17 > 0, A17, 0)", "=SUM(A17, B17)", "=C17+1"], + ["-3", "=IF(A18 > 0, A18, 0)", "=SUM(A18, B18)", "=C18+1"], + ["-2", "=IF(A19 > 0, A19, 0)", "=SUM(A19, B19)", "=C19+1"], + ["-1", "=IF(A20 > 0, A20, 0)", "=SUM(A20, B20)", "=C20+1"], + ["0", "=IF(A21 > 0, A21, 0)", "=SUM(A21, B21)", "=C21+1"], + ["1", "=IF(A22 > 0, A22, 0)", "=SUM(A22, B22)", "=C22+1"], + ["2", "=IF(A23 > 0, A23, 0)", "=SUM(A23, B23)", "=C23+1"], + ["3", "=IF(A24 > 0, A24, 0)", "=SUM(A24, B24)", "=C24+1"], + ["4", "=IF(A25 > 0, A25, 0)", "=SUM(A25, B25)", "=C25+1"], + ["5", "=IF(A26 > 0, A26, 0)", "=SUM(A26, B26)", "=C26+1"], + ["6", "=IF(A27 > 0, A27, 0)", "=SUM(A27, B27)", "=C27+1"], + ["7", "=IF(A28 > 0, A28, 0)", "=SUM(A28, B28)", "=C28+1"], + ["8", "=IF(A29 > 0, A29, 0)", "=SUM(A29, B29)", "=C29+1"], + ["9", "=IF(A30 > 0, A30, 0)", "=SUM(A30, B30)", "=C30+1"], +] + +const typeSizes = { + "undefined": () => 0, + "boolean": () => 4, + "number": () => 8, + "string": item => 2 * item.length, + "object": item => !item ? 0 : Object + .keys(item) + .reduce((total, key) => sizeof(key) + sizeof(item[key]) + total, 0) + }; + +const sizeof = value => typeSizes[typeof value](value); + +console.log("Size of formula matrix:", sizeof(formMatrix)) + +function trim(s, c) { + if (c === "]") c = "\\]"; + if (c === "^") c = "\\^"; + if (c === "\\") c = "\\\\"; + return s.replace(new RegExp( + "^[" + c + "]+|[" + c + "]+$", "g" + ), ""); +} + +function isNumber(c) { + if (typeof c !== 'string') { + return false; + } + + if (c.trim() === '') { + return false; + } + + return false;//!isNaN(c); +} + +function compressFormulaMatrix(formMatrix, pattern) { + if (pattern.substring(0, 3) === "") { + pattern = pattern.substring(4, pattern.length - 1) + } + + let patterns = pattern.split(";") + let result = { 0: pattern } + + for (let i = 0; i < patterns.length; i += 1) { + patterns[i] = patterns[i].split("`") + patterns[i] = patterns[i].filter(s => { return s.length !== 0 }) + } + + for (let row = 0; row < formMatrix.length; row += 1) { + for (let col = 0; col < formMatrix[row].length; col += 1) { + let p = patterns[col] + let f = formMatrix[row][col] + for (let i = 0; i < p.length; i += 1) { + if (p[i].startsWith(", , 0, ,, , A, 16, ,, , 0, ), ;, =, SUM, (, A, 16, ,, , B, 16, ), ;, =, C, 16, +, 1) +ArrayBuffer(-, 4, ;, =, IF, (, A, 17, , >, , 0, ,, , A, 17, ,, , 0, ), ;, =, SUM, (, A, 17, ,, , B, 17, ), ;, =, C, 17, +, 1) +ArrayBuffer(-, 3, ;, =, IF, (, A, 18, , >, , 0, ,, , A, 18, ,, , 0, ), ;, =, SUM, (, A, 18, ,, , B, 18, ), ;, =, C, 18, +, 1) +ArrayBuffer(-, 2, ;, =, IF, (, A, 19, , >, , 0, ,, , A, 19, ,, , 0, ), ;, =, SUM, (, A, 19, ,, , B, 19, ), ;, =, C, 19, +, 1) +ArrayBuffer(-, 1, ;, =, IF, (, A, 20, , >, , 0, ,, , A, 20, ,, , 0, ), ;, =, SUM, (, A, 20, ,, , B, 20, ), ;, =, C, 20, +, 1) +ArrayBuffer(0, ;, =, IF, (, A, 21, , >, , 0, ,, , A, 21, ,, , 0, ), ;, =, SUM, (, A, 21, ,, , B, 21, ), ;, =, C, 21, +, 1) +ArrayBuffer(1, ;, =, IF, (, A, 22, , >, , 0, ,, , A, 22, ,, , 0, ), ;, =, SUM, (, A, 22, ,, , B, 22, ), ;, =, C, 22, +, 1) +ArrayBuffer(2, ;, =, IF, (, A, 23, , >, , 0, ,, , A, 23, ,, , 0, ), ;, =, SUM, (, A, 23, ,, , B, 23, ), ;, =, C, 23, +, 1) +ArrayBuffer(3, ;, =, IF, (, A, 24, , >, , 0, ,, , A, 24, ,, , 0, ), ;, =, SUM, (, A, 24, ,, , B, 24, ), ;, =, C, 24, +, 1) +ArrayBuffer(4, ;, =, IF, (, A, 25, , >, , 0, ,, , A, 25, ,, , 0, ), ;, =, SUM, (, A, 25, ,, , B, 25, ), ;, =, C, 25, +, 1) +ArrayBuffer(5, ;, =, IF, (, A, 26, , >, , 0, ,, , A, 26, ,, , 0, ), ;, =, SUM, (, A, 26, ,, , B, 26, ), ;, =, C, 26, +, 1) +ArrayBuffer(6, ;, =, IF, (, A, 27, , >, , 0, ,, , A, 27, ,, , 0, ), ;, =, SUM, (, A, 27, ,, , B, 27, ), ;, =, C, 27, +, 1) +ArrayBuffer(7, ;, =, IF, (, A, 28, , >, , 0, ,, , A, 28, ,, , 0, ), ;, =, SUM, (, A, 28, ,, , B, 28, ), ;, =, C, 28, +, 1) +ArrayBuffer(8, ;, =, IF, (, A, 29, , >, , 0, ,, , A, 29, ,, , 0, ), ;, =, SUM, (, A, 29, ,, , B, 29, ), ;, =, C, 29, +, 1) +ArrayBuffer(9, ;, =, IF, (, A, 30, , >, , 0, ,, , A, 30, ,, , 0, ), ;, =, SUM, (, A, 30, ,, , B, 30, ), ;, =, C, 30, +, 1) \ No newline at end of file diff --git a/add-in/src/api/pids/pattern/extractPattern.ts b/add-in/src/api/pids/pattern/extractPattern.ts new file mode 100644 index 0000000..d374f5c --- /dev/null +++ b/add-in/src/api/pids/pattern/extractPattern.ts @@ -0,0 +1,32 @@ +import { PatternMiner } from "./patternMiner"; + +/* + Extract a pattern from a formula matrix that is stored in row-major. + Returns an array of patterns corresponding to each column. +*/ +export function extractPattern(formMatrix) { + let miner = new PatternMiner(); + miner.sampleSize = 2000; + let transpose = transposeArray(formMatrix); + let patterns = []; + for (let colIdx = 0; colIdx < transpose.length; colIdx += 1) { + let pattern = miner.mine(transpose[colIdx]); + patterns.push(pattern); + console.log(pattern); + } + return patterns; +} + +/* + Transpose the array to change it from row to column major. + We evaluate PIDS along each column. +*/ +function transposeArray(formMatrix) { + let result = JSON.parse(JSON.stringify(formMatrix)); + for (let i = 0; i < formMatrix.length; i += 1) { + for (let j = 0; j < formMatrix[i].length; j += 1) { + result[j][i] = formMatrix[i][j]; + } + } + return result; +} diff --git a/add-in/src/api/pids/pattern/pattern.ts b/add-in/src/api/pids/pattern/pattern.ts new file mode 100644 index 0000000..b6f30a0 --- /dev/null +++ b/add-in/src/api/pids/pattern/pattern.ts @@ -0,0 +1,27 @@ +abstract class Pattern { + name: string; + + getName() { + return name; + } + + /** + * @return all leaf patterns + */ + flatten() { + // return Seq(this) + } + + /** + * Recursively visit the pattern elements starting from the root + * + * @param visitor + */ + abstract visit(visitor); + + naming() { + // this.visit(new NamingVisitor()) + } + + abstract numChar(); +} diff --git a/add-in/src/api/pids/pattern/patternMiner.ts b/add-in/src/api/pids/pattern/patternMiner.ts new file mode 100644 index 0000000..79e06c9 --- /dev/null +++ b/add-in/src/api/pids/pattern/patternMiner.ts @@ -0,0 +1,13 @@ +import { Tokenizer } from "../tokenize/tokenizer"; + +export class PatternMiner { + sampleSize = 500; + + rules = []; // fill in with rules later + + mine(lines) { + let tokens = lines.map((line) => Tokenizer.tokenize(line)); + } + + refine() {} +} diff --git a/add-in/src/api/pids/tests/extractionTest.ts b/add-in/src/api/pids/tests/extractionTest.ts new file mode 100644 index 0000000..3d1d64c --- /dev/null +++ b/add-in/src/api/pids/tests/extractionTest.ts @@ -0,0 +1,4 @@ +import { testFormMatrix } from "./testMatrix"; +import { extractPattern } from "../pattern/extractPattern"; + +extractPattern(testFormMatrix); diff --git a/add-in/src/api/pids/tests/testMatrix.ts b/add-in/src/api/pids/tests/testMatrix.ts new file mode 100644 index 0000000..b7e12a5 --- /dev/null +++ b/add-in/src/api/pids/tests/testMatrix.ts @@ -0,0 +1,17 @@ +export const testFormMatrix = [ + ["-5", "=IF(A16 > 0, A16, 0)", "=SUM(A16, B16)", "=C16+1"], + ["-4", "=IF(A17 > 0, A17, 0)", "=SUM(A17, B17)", "=C17+1"], + ["-3", "=IF(A18 > 0, A18, 0)", "=SUM(A18, B18)", "=C18+1"], + ["-2", "=IF(A19 > 0, A19, 0)", "=SUM(A19, B19)", "=C19+1"], + ["-1", "=IF(A20 > 0, A20, 0)", "=SUM(A20, B20)", "=C20+1"], + ["0", "=IF(A21 > 0, A21, 0)", "=SUM(A21, B21)", "=C21+1"], + ["1", "=IF(A22 > 0, A22, 0)", "=SUM(A22, B22)", "=C22+1"], + ["2", "=IF(A23 > 0, A23, 0)", "=SUM(A23, B23)", "=C23+1"], + ["3", "=IF(A24 > 0, A24, 0)", "=SUM(A24, B24)", "=C24+1"], + ["4", "=IF(A25 > 0, A25, 0)", "=SUM(A25, B25)", "=C25+1"], + ["5", "=IF(A26 > 0, A26, 0)", "=SUM(A26, B26)", "=C26+1"], + ["6", "=IF(A27 > 0, A27, 0)", "=SUM(A27, B27)", "=C27+1"], + ["7", "=IF(A28 > 0, A28, 0)", "=SUM(A28, B28)", "=C28+1"], + ["8", "=IF(A29 > 0, A29, 0)", "=SUM(A29, B29)", "=C29+1"], + ["9", "=IF(A30 > 0, A30, 0)", "=SUM(A30, B30)", "=C30+1"], +]; diff --git a/add-in/src/api/pids/tests/tokenizeTest.ts b/add-in/src/api/pids/tests/tokenizeTest.ts new file mode 100644 index 0000000..f966caf --- /dev/null +++ b/add-in/src/api/pids/tests/tokenizeTest.ts @@ -0,0 +1,4 @@ +import { testFormMatrix } from "./testMatrix"; +import { Tokenizer } from "../tokenize/tokenizer"; + +console.log(Tokenizer.tokenize(testFormMatrix[0])); diff --git a/add-in/src/api/pids/tokenize/lexer.ts b/add-in/src/api/pids/tokenize/lexer.ts new file mode 100644 index 0000000..d085543 --- /dev/null +++ b/add-in/src/api/pids/tokenize/lexer.ts @@ -0,0 +1,82 @@ +import { lexer, lexerResults } from "js-lexer"; +import { TWord, TInt, TDouble, TSymbol, TSpace } from "./token"; + +const conds = [ + { + name: "word", + tester: function (tested) { + var regResult = tested.match(/[a-zA-Z]+/); + if (regResult && regResult[0] === tested) { + return lexerResults.possible; + } else { + return lexerResults.none; + } + }, + }, + // { + // name: "double", + // tester: function (tested) { + // var regResult = tested.match(/[+-]?([0-9]*[.])?[0-9]+/); + // if (regResult && regResult[0] === tested) { + // return lexerResults.possible; + // } else { + // return lexerResults.none; + // } + // }, + // }, + { + name: "int", + tester: function (tested) { + var regResult = tested.match(/\d+/); + if (regResult && regResult[0] === tested) { + return lexerResults.possible; + } else { + return lexerResults.none; + } + }, + }, + { + name: "space", + tester: function (tested) { + if (tested === " ") { + return lexerResults.exact; + } else { + return lexerResults.none; + } + }, + }, + { + name: "symbol", + tester: function (tested) { + var regResult = tested.match(/[^a-zA-Z0-9]/g); + if (regResult && regResult[0] === tested) { + return lexerResults.possible; + } else { + return lexerResults.none; + } + }, + }, +]; + +export class Lexer { + static lex(str) { + let tokens = lexer(str, conds); + return tokens.map((token) => { + switch (token.type) { + case "word": + return new TWord(token.value); + case "int": + return new TInt(parseInt(token.value)); + case "double": + return new TDouble(parseFloat(token.value)); + case "symbol": + return new TSymbol(token.value); + case "space": + return new TSpace(); + default: + console.log("Token type does not exist!"); + return new TSymbol(""); + } + }); + } +} diff --git a/add-in/src/api/pids/tokenize/token.ts b/add-in/src/api/pids/tokenize/token.ts new file mode 100644 index 0000000..383fe07 --- /dev/null +++ b/add-in/src/api/pids/tokenize/token.ts @@ -0,0 +1,71 @@ +import BigDecimal from "bigdecimal"; +import { hashCode } from "../utils/hash"; + +abstract class Token { + value: string; + isData = false; + length = 1; + numChar = 1; + + toString = () => this.value.toString(); + + canEqual(other) { + return typeof other === typeof this; + } + + equals(other) { + if (other instanceof Token) { + return other.canEqual(this) && this.value == other.value; + } + return false; + } + + hashCode = () => hashCode(this.value); +} + +export class TWord extends Token { + constructor(v) { + super(); + this.value = v.toString(); + this.isData = true; + this.length = this.value.length; + } +} + +export class TInt extends Token { + constructor(v) { + super(); + this.value = v.toString(); + this.isData = true; + this.intValue = BigInt(v); + } + + intValue: BigInt; + length = 4; +} + +export class TDouble extends Token { + constructor(v) { + super(); + this.value = v.toString(); + this.isData = true; + this.doubleValue = BigDecimal(this.value); + this.length = this.doubleValue.toDouble().toFloat() ? 4 : 8; + } + + doubleValue: BigDecimal; + length = 4; +} + +export class TSymbol extends Token { + constructor(v) { + super(); + this.value = v.toString(); + } +} + +export class TSpace extends TSymbol { + constructor() { + super(" "); + } +} diff --git a/add-in/src/api/pids/tokenize/tokenizer.ts b/add-in/src/api/pids/tokenize/tokenizer.ts new file mode 100644 index 0000000..13852f8 --- /dev/null +++ b/add-in/src/api/pids/tokenize/tokenizer.ts @@ -0,0 +1,11 @@ +import { Lexer } from "./lexer"; + +export class Tokenizer { + static tokenize(line) { + let tokens = []; + for (let i = 0; i < line.length; i += 1) { + tokens.push(...Lexer.lex(line[i])); + } + return tokens; + } +} diff --git a/add-in/src/api/pids/utils/hash.ts b/add-in/src/api/pids/utils/hash.ts new file mode 100644 index 0000000..eeece10 --- /dev/null +++ b/add-in/src/api/pids/utils/hash.ts @@ -0,0 +1,12 @@ +export function hashCode(str) { + var hash = 0, + i, + chr; + if (str.length === 0) return hash; + for (i = 0; i < this.length; i++) { + chr = str.charCodeAt(i); + hash = (hash << 5) - hash + chr; + hash |= 0; // Convert to 32bit integer + } + return hash; +} diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..48e341a --- /dev/null +++ b/package-lock.json @@ -0,0 +1,3 @@ +{ + "lockfileVersion": 1 +} From 27186f88b588acfa158532c8dc6d7dc30a85e329 Mon Sep 17 00:00:00 2001 From: Connor Lien Date: Mon, 4 Jul 2022 18:02:08 -0700 Subject: [PATCH 2/9] Common Symbol Rule --- add-in/src/api/pids/pattern/pattern.ts | 179 +++++++++++++++++- add-in/src/api/pids/pattern/patternMiner.ts | 32 +++- add-in/src/api/pids/rule/commonSeq.ts | 78 ++++++++ add-in/src/api/pids/rule/commonSymbolRule.ts | 134 +++++++++++++ add-in/src/api/pids/rule/rewriteRule.ts | 46 +++++ .../pids/{tests => test}/extractionTest.ts | 0 add-in/src/api/pids/test/minerTest.ts | 5 + .../api/pids/{tests => test}/testMatrix.ts | 2 +- add-in/src/api/pids/test/tokenizeTest.ts | 5 + add-in/src/api/pids/tests/tokenizeTest.ts | 4 - add-in/src/api/pids/tokenize/token.ts | 2 +- add-in/src/api/pids/utils/stringify.ts | 15 ++ add-in/tsconfig.json | 19 +- 13 files changed, 493 insertions(+), 28 deletions(-) create mode 100644 add-in/src/api/pids/rule/commonSeq.ts create mode 100644 add-in/src/api/pids/rule/commonSymbolRule.ts create mode 100644 add-in/src/api/pids/rule/rewriteRule.ts rename add-in/src/api/pids/{tests => test}/extractionTest.ts (100%) create mode 100644 add-in/src/api/pids/test/minerTest.ts rename add-in/src/api/pids/{tests => test}/testMatrix.ts (96%) create mode 100644 add-in/src/api/pids/test/tokenizeTest.ts delete mode 100644 add-in/src/api/pids/tests/tokenizeTest.ts create mode 100644 add-in/src/api/pids/utils/stringify.ts diff --git a/add-in/src/api/pids/pattern/pattern.ts b/add-in/src/api/pids/pattern/pattern.ts index b6f30a0..4c74ee8 100644 --- a/add-in/src/api/pids/pattern/pattern.ts +++ b/add-in/src/api/pids/pattern/pattern.ts @@ -1,15 +1,17 @@ -abstract class Pattern { +import { Token } from "../tokenize/token"; + +export abstract class Pattern { name: string; getName() { - return name; + return this.name; } /** * @return all leaf patterns */ flatten() { - // return Seq(this) + return [this]; } /** @@ -17,11 +19,174 @@ abstract class Pattern { * * @param visitor */ - abstract visit(visitor); + visit = (visitor) => { + visitor.on(this); + }; + + abstract numChar(); +} - naming() { - // this.visit(new NamingVisitor()) +export class PToken extends Pattern { + token: Token; + + constructor(token) { + super(); + this.token = token; } - abstract numChar(); + numChar() { + return this.token.numChar, this.token.numChar; + } + + canEqual = (other) => other instanceof PToken; + + equals(other) { + if (other instanceof PToken) { + return other.canEqual(this) && this.token == other.token; + } + return false; + } + + override toString = () => this.token.toString(); +} + +export class PSeq extends Pattern { + content = []; + + constructor(contents) { + super(); + this.content = contents; + } + + flatten = () => this.content.map((x) => x.flatten()); + + visit = (visitor) => { + visitor.on(this); + visitor.enter(this); + this.content.map((x) => x.visit(visitor)); + visitor.exit(this); + }; + + numChar = () => { + this.content.map((x) => x.numChar()).reduce((a, b) => a + b); + }; + + canEqual = (other) => other instanceof PSeq; + + equals = (other) => { + if (other instanceof PSeq) { + return other.canEqual(this) && this.content == other.content; + } + return false; + }; + + override toString = () => `(${this.content.map((x) => x.toString()).join(",")})`; +} + +export class PUnion extends Pattern { + content = []; + + constructor(content) { + super(); + this.content = content; + } + + flatten = () => this.content.map((x) => x.flatten()); + + visit = (visitor) => { + visitor.on(this); + visitor.enter(this); + this.content.map((x) => x.visit(visitor)); + visitor.exit(this); + }; + + numChar = () => { + this.content.map((x) => x.numChar()).reduce((a, b) => Math.min(a, b) + Math.max(a, b)); + }; + + canEqual = (other) => other instanceof PUnion; + + equals = (other) => { + if (other instanceof PUnion) { + return other.canEqual(PUnion) && this.content == other.content; + } + return false; + }; + + override toString = () => `(\n${this.content.map((x) => x.toString()).join(",\n")}\n)`; +} + +export class PEmpty extends Pattern { + numChar = () => [0, 0]; + + override toString = () => ""; +} + +abstract class PAny extends Pattern { + minLength; + maxLength; + + constructor(minLength, maxLength) { + super(); + this.minLength = minLength; + this.maxLength = maxLength; + } + + numChar = () => [this.minLength, this.maxLength]; + + canEqual(other) { + return typeof other === typeof this; + } + + equals(other) { + if (other instanceof PAny) { + return other.canEqual(PAny) && this.minLength == other.minLength && this.maxLength == other.maxLength; + } + return false; + } +} + +class PIntAny extends PAny { + hasHex = false; + + constructor(minl = 1, maxl = -1, hasHex = false) { + super(minl, maxl); + this.hasHex = hasHex; + } + + equals(other) { + if (other instanceof PIntAny) { + return this.canEqual(other) && this.minLength == other.minLength && this.maxLength == other.maxLength; + } + return false; + } + + override toString = () => ``; +} + +class PLetterAny extends PAny { + constructor(minl = 1, maxl = -1) { + super(minl, maxl); + } + + override toString = () => ``; +} + +/** + * Mix of letter and digits + */ +class PLabelAny extends PAny { + constructor(minl = 1, maxl = -1) { + super(minl, maxl); + } + + override toString = () => ``; +} + +class PWordAny extends PAny { + constructor(minl = 1, maxl = -1) { + super(minl, maxl); + } + + override toString = () => ``; } diff --git a/add-in/src/api/pids/pattern/patternMiner.ts b/add-in/src/api/pids/pattern/patternMiner.ts index 79e06c9..7af870e 100644 --- a/add-in/src/api/pids/pattern/patternMiner.ts +++ b/add-in/src/api/pids/pattern/patternMiner.ts @@ -1,13 +1,41 @@ import { Tokenizer } from "../tokenize/tokenizer"; +import { Pattern, PUnion, PSeq, PToken } from "./pattern"; +import { CommonSymbolRule } from "../rule/commonSymbolRule"; export class PatternMiner { sampleSize = 500; - rules = []; // fill in with rules later + rules = [new CommonSymbolRule()]; // fill in with rules later mine(lines) { let tokens = lines.map((line) => Tokenizer.tokenize(line)); + let translated = new PUnion(tokens.map((line) => new PSeq(line.map((elem) => new PToken(elem))))); + + let toRefine: Pattern = translated; + let needRefine = true; + let refineResult: Pattern = toRefine; + while (needRefine) { + let refined = this.refine(toRefine); + if (refined[1] != null) { + toRefine = refined[0]; + } else { + needRefine = false; + refineResult = refined[0]; + } + } + return refineResult; } - refine() {} + refine(root) { + let current = root; + for (let i = 0; i < this.rules.length; i += 1) { + const rule = this.rules[i]; + rule.reset(); + current = rule.rewrite(current); + if (rule.happened) { + return [current, rule]; + } + } + return [root, null]; + } } diff --git a/add-in/src/api/pids/rule/commonSeq.ts b/add-in/src/api/pids/rule/commonSeq.ts new file mode 100644 index 0000000..2e6e40c --- /dev/null +++ b/add-in/src/api/pids/rule/commonSeq.ts @@ -0,0 +1,78 @@ +export class CommonSeq { + bool2int = (b: Boolean) => (b ? 1 : 0); + + /** + * Look for common sequence in a list of lines. For implementation + * simplicity, only the longest common seq is returned + * + * @param lines + * @return common sequences + */ + find(lines: any[][], equal) { + let commons = lines[0]; + + if (lines.length == 1) { + return commons; + } + lines.slice(1).forEach((line) => { + if (commons.length !== 0) { + let commonsBetween = this.between(commons, line, equal); + // TODO: This is a placeholder. Refer to line 84 in CommonSeq.scala. + commons = commonsBetween; + } + }); + return commons; + } + + /** + * Find the longest common subsequence in two arrays + * + * Algorithm from https://www.geeksforgeeks.org/printing-longest-common-subsequence/ + * + * @param a the first array + * @param b the second array + * @param equal equality test function + * @return array of common symbols + */ + between(a: any[], b: any[], equal: (a: any, b: any) => Boolean): any[] { + // Find the longest common in order subsequences + let len1 = a.length; + let len2 = b.length; + let lcs = new Array(len1 + 1); + for (let i = 0; i <= len1; i += 1) { + lcs[i] = new Array(len2 + 1); + } + for (let i = 0; i <= len1; i += 1) { + for (let j = 0; j <= len2; j += 1) { + if (i == 0 || j == 0) { + lcs[i][j] = 0; + } else { + if (equal(a[i - 1], b[j - 1])) { + lcs[i][j] = 1 + lcs[i - 1][j - 1]; + } else { + lcs[i][j] = Math.max(lcs[i][j - 1], lcs[i - 1][j]); + } + } + } + } + + // Reconstruct longest sequence + let result = []; + let i = len1; + let j = len2; + while (i > 0 && j > 0) { + if (equal(a[i - 1], b[j - 1])) { + result.push(a[i - 1]); + i -= 1; + j -= 1; + } else { + if (lcs[i][j - 1] > lcs[i - 1][j]) { + j -= 1; + } else { + i -= 1; + } + } + } + return result.reverse(); + } +} diff --git a/add-in/src/api/pids/rule/commonSymbolRule.ts b/add-in/src/api/pids/rule/commonSymbolRule.ts new file mode 100644 index 0000000..06449ee --- /dev/null +++ b/add-in/src/api/pids/rule/commonSymbolRule.ts @@ -0,0 +1,134 @@ +import { RewriteRule } from "./rewriteRule"; +import { PUnion, PSeq, PToken, PEmpty } from "../pattern/pattern"; +import { TSymbol } from "../tokenize/token"; +import { CommonSeq } from "./commonSeq"; + +export class CommonSymbolRule extends RewriteRule { + static threshold = 0.4; + + condition(ptn) { + if (!(ptn instanceof PUnion)) { + return false; + } + + let cnt = ptn.content; + if (cnt.length <= 1) { + return false; + } + + let res = cnt.map((x) => { + if (x instanceof PSeq) { + return [x.content.every((t) => t instanceof PToken || t == PEmpty), x.content.length]; + } else if (x instanceof PToken) { + return [true, 1]; + } else if (x == PEmpty) { + return [true, 1]; + } else { + return [false, 0]; + } + }); + + return res.some((x) => x[1] > 1) && res.every((x) => x[0]); + } + + update(union: PUnion) { + // Flatten the union content + let unionData = union.content.map((item) => { + let result = [item]; + if (item instanceof PSeq) { + result = item.content; + } else if (item instanceof PEmpty) { + result = []; + } + return result; + }); + if (unionData.length === 1) { + return union; + } + + // Scan union data for symbols + let symbolsWithPos = unionData.map((line) => + line + .map((value, index) => [value, index]) + .filter((value) => { + if (value[0] instanceof PToken) { + return value[0].token instanceof TSymbol; + } + return false; + }) + ); + + // Valid lines have at least one symbol + let noSymbolLines = new Set(); + let validLinesWithIndex = new Set>(); + symbolsWithPos + .map((value, idx) => [value, idx]) + .forEach((value) => { + if ((value[0] as Array).length === 0) { + noSymbolLines.add(value[1] as number); + } else { + validLinesWithIndex.add(value); + } + }); + + if (validLinesWithIndex.size === 0) { + return union; + } + + // If there is at least one line with no symbol, the common symbol found is optional + let optionalSymbol = noSymbolLines.size !== 0; + let validLines = Array.from(validLinesWithIndex).map((value) => value[0]); + let validIndexMapping = Array.from(validLinesWithIndex) + .map((value) => value[1]) + .map((value, idx) => [value, idx]) + .reduce((acc, val) => { + const [key, value] = val; + acc[key] = value; + return acc; + }, {}); + + // Determine common symbols and match symbol in each line to the common + let commonSeq = new CommonSeq(); + const eq = (a: Array, b: Array) => (a[0] as PToken).toString() === (b[0] as PToken).toString(); + let commonSymbols = commonSeq.find(validLines, eq).map((i) => i[0]); + + // Filter out symbols that are not common + let commonSymbolsWithPos = []; + symbolsWithPos.forEach((line) => { + let idx = 0; + let commonSymbolLine = []; + commonSymbols.forEach((symbol) => { + while (symbol.token.value !== line[idx][0].token.value) { + idx += 1; + } + commonSymbolLine.push(line[idx]); + }); + commonSymbolsWithPos.push(commonSymbolLine); + }); + + // Package new union split by common symbols + let resultSeq = []; + commonSymbols.forEach((symbol, symbolIdx) => { + let beforeSymbolUnion = []; + let nonEmpty: boolean = false; + commonSymbolsWithPos.forEach((line, lineIdx) => { + let startIdx = symbolIdx == 0 ? 0 : line[symbolIdx - 1][1] + 1; + let endIdx = line[symbolIdx][1]; + const unionLine: any[] = union.content[lineIdx].content; + if (endIdx - startIdx > 0) { + let unit = unionLine.slice(startIdx, endIdx); + beforeSymbolUnion.push(new PSeq(unit)); + nonEmpty = true; + } else { + beforeSymbolUnion.push(new PEmpty()); + } + }); + if (nonEmpty) { + resultSeq.push(new PUnion(beforeSymbolUnion)); + } + resultSeq.push(symbol); + }); + + return new PSeq(resultSeq); + } +} diff --git a/add-in/src/api/pids/rule/rewriteRule.ts b/add-in/src/api/pids/rule/rewriteRule.ts new file mode 100644 index 0000000..5520767 --- /dev/null +++ b/add-in/src/api/pids/rule/rewriteRule.ts @@ -0,0 +1,46 @@ +import { Pattern, PUnion, PSeq } from "../pattern/pattern"; + +export abstract class RewriteRule { + modified = false; + + path = []; + + happen = () => (this.modified = true); + + happened = this.modified; + + reset = () => (this.modified = false); + + rewrite(root) { + this.path.push(root); + let result = null; + + if (root instanceof PUnion) { + let union: PUnion = root; + if (this.condition(union)) { + this.update(union); + } else { + let modifiedContent = union.content.map((p) => this.rewrite(p)); + result = this.modified ? new PUnion(modifiedContent) : union; + } + } else if (root instanceof PSeq) { + let seq: PSeq = root; + if (this.condition(seq)) { + this.update(seq); + } else { + let modifiedContent = seq.content.map((p) => this.rewrite(p)); + result = this.modified ? new PSeq(modifiedContent) : seq; + } + } else { + let pattern: Pattern = root; + result = this.condition(pattern) ? this.update(pattern) : pattern; + } + + this.path.pop(); + return result; + } + + abstract condition(ptn); + + abstract update(ptn); +} diff --git a/add-in/src/api/pids/tests/extractionTest.ts b/add-in/src/api/pids/test/extractionTest.ts similarity index 100% rename from add-in/src/api/pids/tests/extractionTest.ts rename to add-in/src/api/pids/test/extractionTest.ts diff --git a/add-in/src/api/pids/test/minerTest.ts b/add-in/src/api/pids/test/minerTest.ts new file mode 100644 index 0000000..6c2c8a2 --- /dev/null +++ b/add-in/src/api/pids/test/minerTest.ts @@ -0,0 +1,5 @@ +import { testFormMatrix1 } from "./testMatrix"; +import { PatternMiner } from "../pattern/patternMiner"; + +let miner = new PatternMiner(); +console.log(miner.mine(testFormMatrix1).toString()); diff --git a/add-in/src/api/pids/tests/testMatrix.ts b/add-in/src/api/pids/test/testMatrix.ts similarity index 96% rename from add-in/src/api/pids/tests/testMatrix.ts rename to add-in/src/api/pids/test/testMatrix.ts index b7e12a5..b62eb17 100644 --- a/add-in/src/api/pids/tests/testMatrix.ts +++ b/add-in/src/api/pids/test/testMatrix.ts @@ -1,4 +1,4 @@ -export const testFormMatrix = [ +export const testFormMatrix1 = [ ["-5", "=IF(A16 > 0, A16, 0)", "=SUM(A16, B16)", "=C16+1"], ["-4", "=IF(A17 > 0, A17, 0)", "=SUM(A17, B17)", "=C17+1"], ["-3", "=IF(A18 > 0, A18, 0)", "=SUM(A18, B18)", "=C18+1"], diff --git a/add-in/src/api/pids/test/tokenizeTest.ts b/add-in/src/api/pids/test/tokenizeTest.ts new file mode 100644 index 0000000..eee3294 --- /dev/null +++ b/add-in/src/api/pids/test/tokenizeTest.ts @@ -0,0 +1,5 @@ +import { testFormMatrix } from "./testMatrix"; +import { stringifyArray } from "../utils/stringify"; +import { Tokenizer } from "../tokenize/tokenizer"; + +console.log(stringifyArray(Tokenizer.tokenize(testFormMatrix[0]))); diff --git a/add-in/src/api/pids/tests/tokenizeTest.ts b/add-in/src/api/pids/tests/tokenizeTest.ts deleted file mode 100644 index f966caf..0000000 --- a/add-in/src/api/pids/tests/tokenizeTest.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { testFormMatrix } from "./testMatrix"; -import { Tokenizer } from "../tokenize/tokenizer"; - -console.log(Tokenizer.tokenize(testFormMatrix[0])); diff --git a/add-in/src/api/pids/tokenize/token.ts b/add-in/src/api/pids/tokenize/token.ts index 383fe07..a44a03a 100644 --- a/add-in/src/api/pids/tokenize/token.ts +++ b/add-in/src/api/pids/tokenize/token.ts @@ -1,7 +1,7 @@ import BigDecimal from "bigdecimal"; import { hashCode } from "../utils/hash"; -abstract class Token { +export abstract class Token { value: string; isData = false; length = 1; diff --git a/add-in/src/api/pids/utils/stringify.ts b/add-in/src/api/pids/utils/stringify.ts new file mode 100644 index 0000000..619b03c --- /dev/null +++ b/add-in/src/api/pids/utils/stringify.ts @@ -0,0 +1,15 @@ +export function stringifyArray(arr) { + let result = []; + for (let i = 0; i < arr.length; i += 1) { + result.push(arr[i].toString()); + } + return result; +} + +export function stringifyMatrix(mat) { + let result = []; + for (let i = 0; i < mat.length; i += 1) { + result.push(stringifyArray(mat[i])); + } + return result; +} diff --git a/add-in/tsconfig.json b/add-in/tsconfig.json index 1800f1c..32c9c65 100644 --- a/add-in/tsconfig.json +++ b/add-in/tsconfig.json @@ -13,21 +13,14 @@ "noImplicitReturns": true, "noUnusedParameters": true, "esModuleInterop": true, - "lib": [ - "es7", - "dom" - ], + "lib": ["es7", "dom", "es2019"], "pretty": true, - "typeRoots": [ - "node_modules/@types" - ] + "typeRoots": ["node_modules/@types"] }, - "exclude": [ - "node_modules" - ], + "exclude": ["node_modules"], "compileOnSave": false, "buildOnSave": false, - "ts-node": { - "files": true + "ts-node": { + "files": true } -} \ No newline at end of file +} From d7a1230c1a8cd551d4e08aba461e93827e393381 Mon Sep 17 00:00:00 2001 From: Connor Lien Date: Thu, 25 Aug 2022 17:39:31 -0700 Subject: [PATCH 3/9] Rewrite rule --- add-in/src/api/pids/pattern/pattern.ts | 8 ++-- add-in/src/api/pids/pattern/patternMiner.ts | 7 ++-- add-in/src/api/pids/rule/commonSymbolRule.ts | 18 ++------ add-in/src/api/pids/rule/rewriteRule.ts | 6 +-- add-in/src/api/pids/rule/sameItemRule.ts | 44 ++++++++++++++++++++ 5 files changed, 58 insertions(+), 25 deletions(-) create mode 100644 add-in/src/api/pids/rule/sameItemRule.ts diff --git a/add-in/src/api/pids/pattern/pattern.ts b/add-in/src/api/pids/pattern/pattern.ts index 4c74ee8..d8de9d7 100644 --- a/add-in/src/api/pids/pattern/pattern.ts +++ b/add-in/src/api/pids/pattern/pattern.ts @@ -146,7 +146,7 @@ abstract class PAny extends Pattern { } } -class PIntAny extends PAny { +export class PIntAny extends PAny { hasHex = false; constructor(minl = 1, maxl = -1, hasHex = false) { @@ -164,7 +164,7 @@ class PIntAny extends PAny { override toString = () => ``; } -class PLetterAny extends PAny { +export class PLetterAny extends PAny { constructor(minl = 1, maxl = -1) { super(minl, maxl); } @@ -175,7 +175,7 @@ class PLetterAny extends PAny { /** * Mix of letter and digits */ -class PLabelAny extends PAny { +export class PLabelAny extends PAny { constructor(minl = 1, maxl = -1) { super(minl, maxl); } @@ -183,7 +183,7 @@ class PLabelAny extends PAny { override toString = () => ``; } -class PWordAny extends PAny { +export class PWordAny extends PAny { constructor(minl = 1, maxl = -1) { super(minl, maxl); } diff --git a/add-in/src/api/pids/pattern/patternMiner.ts b/add-in/src/api/pids/pattern/patternMiner.ts index 7af870e..2cb3a06 100644 --- a/add-in/src/api/pids/pattern/patternMiner.ts +++ b/add-in/src/api/pids/pattern/patternMiner.ts @@ -1,11 +1,12 @@ import { Tokenizer } from "../tokenize/tokenizer"; import { Pattern, PUnion, PSeq, PToken } from "./pattern"; import { CommonSymbolRule } from "../rule/commonSymbolRule"; +import { SameItemRule } from "../rule/sameItemRule"; export class PatternMiner { sampleSize = 500; - rules = [new CommonSymbolRule()]; // fill in with rules later + rules = [new CommonSymbolRule(), new SameItemRule()]; mine(lines) { let tokens = lines.map((line) => Tokenizer.tokenize(line)); @@ -32,10 +33,10 @@ export class PatternMiner { const rule = this.rules[i]; rule.reset(); current = rule.rewrite(current); - if (rule.happened) { + if (rule.modified) { return [current, rule]; } } - return [root, null]; + return [current, null]; } } diff --git a/add-in/src/api/pids/rule/commonSymbolRule.ts b/add-in/src/api/pids/rule/commonSymbolRule.ts index 06449ee..6f84f52 100644 --- a/add-in/src/api/pids/rule/commonSymbolRule.ts +++ b/add-in/src/api/pids/rule/commonSymbolRule.ts @@ -2,10 +2,9 @@ import { RewriteRule } from "./rewriteRule"; import { PUnion, PSeq, PToken, PEmpty } from "../pattern/pattern"; import { TSymbol } from "../tokenize/token"; import { CommonSeq } from "./commonSeq"; +import { stringifyArray, stringifyMatrix } from "../utils/stringify"; export class CommonSymbolRule extends RewriteRule { - static threshold = 0.4; - condition(ptn) { if (!(ptn instanceof PUnion)) { return false; @@ -71,21 +70,11 @@ export class CommonSymbolRule extends RewriteRule { } }); - if (validLinesWithIndex.size === 0) { + if (noSymbolLines.size > 0) { return union; } - // If there is at least one line with no symbol, the common symbol found is optional - let optionalSymbol = noSymbolLines.size !== 0; let validLines = Array.from(validLinesWithIndex).map((value) => value[0]); - let validIndexMapping = Array.from(validLinesWithIndex) - .map((value) => value[1]) - .map((value, idx) => [value, idx]) - .reduce((acc, val) => { - const [key, value] = val; - acc[key] = value; - return acc; - }, {}); // Determine common symbols and match symbol in each line to the common let commonSeq = new CommonSeq(); @@ -98,7 +87,7 @@ export class CommonSymbolRule extends RewriteRule { let idx = 0; let commonSymbolLine = []; commonSymbols.forEach((symbol) => { - while (symbol.token.value !== line[idx][0].token.value) { + while (line[idx] && symbol.token.value !== line[idx][0].token.value) { idx += 1; } commonSymbolLine.push(line[idx]); @@ -108,6 +97,7 @@ export class CommonSymbolRule extends RewriteRule { // Package new union split by common symbols let resultSeq = []; + this.happen(); commonSymbols.forEach((symbol, symbolIdx) => { let beforeSymbolUnion = []; let nonEmpty: boolean = false; diff --git a/add-in/src/api/pids/rule/rewriteRule.ts b/add-in/src/api/pids/rule/rewriteRule.ts index 5520767..db4f382 100644 --- a/add-in/src/api/pids/rule/rewriteRule.ts +++ b/add-in/src/api/pids/rule/rewriteRule.ts @@ -7,8 +7,6 @@ export abstract class RewriteRule { happen = () => (this.modified = true); - happened = this.modified; - reset = () => (this.modified = false); rewrite(root) { @@ -18,7 +16,7 @@ export abstract class RewriteRule { if (root instanceof PUnion) { let union: PUnion = root; if (this.condition(union)) { - this.update(union); + result = this.update(union); } else { let modifiedContent = union.content.map((p) => this.rewrite(p)); result = this.modified ? new PUnion(modifiedContent) : union; @@ -26,7 +24,7 @@ export abstract class RewriteRule { } else if (root instanceof PSeq) { let seq: PSeq = root; if (this.condition(seq)) { - this.update(seq); + result = this.update(seq); } else { let modifiedContent = seq.content.map((p) => this.rewrite(p)); result = this.modified ? new PSeq(modifiedContent) : seq; diff --git a/add-in/src/api/pids/rule/sameItemRule.ts b/add-in/src/api/pids/rule/sameItemRule.ts new file mode 100644 index 0000000..780a93e --- /dev/null +++ b/add-in/src/api/pids/rule/sameItemRule.ts @@ -0,0 +1,44 @@ +import { RewriteRule } from "./rewriteRule"; +import { PUnion, PSeq, PToken, PEmpty, Pattern } from "../pattern/pattern"; + +export class SameItemRule extends RewriteRule { + condition(ptn) { + if (ptn instanceof PUnion || ptn instanceof PSeq) { + return true; + } + + return false; + } + + update(ptn: Pattern) { + if (ptn instanceof PSeq) { + let result = ptn.content.map((cont) => { + if (cont instanceof PUnion) { + return this.combineUnion(cont); + } else { + return cont; + } + }); + return result; + } + return ptn; + } + + combineUnion(ptn: PUnion) { + let content = ptn.content; + if (content.length === 0) { + return new PEmpty(); + } + let allEqual: Boolean = true; + let first = content[0].content; + content.forEach((seq) => { + if (seq.content.toString() !== first.toString()) { + allEqual = false; + } + }); + if (allEqual) { + return new PToken(first); + } + return ptn; + } +} From 006da19c6162f93be25abe4650b4b223969327d1 Mon Sep 17 00:00:00 2001 From: Connor Lien Date: Thu, 25 Aug 2022 17:40:47 -0700 Subject: [PATCH 4/9] Remove archive --- add-in/src/api/pids/archive/lexer.ts | 320 --------------------- add-in/src/api/pids/archive/parsingTest.ts | 90 ------ add-in/src/api/pids/archive/testTokens.txt | 15 - 3 files changed, 425 deletions(-) delete mode 100644 add-in/src/api/pids/archive/lexer.ts delete mode 100644 add-in/src/api/pids/archive/parsingTest.ts delete mode 100644 add-in/src/api/pids/archive/testTokens.txt diff --git a/add-in/src/api/pids/archive/lexer.ts b/add-in/src/api/pids/archive/lexer.ts deleted file mode 100644 index 6e7a00b..0000000 --- a/add-in/src/api/pids/archive/lexer.ts +++ /dev/null @@ -1,320 +0,0 @@ -// /** This character denotes the end of file */ -// const YYEOF = -1; - -// /** initial size of the lookahead buffer */ -// const ZZ_BUFFERSIZE = 16384; - -// /** lexical states */ -// const YYINITIAL = 0; - -// /** -// * ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l -// * ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l -// * at the beginning of a line -// * l is of the form l = 2*k, k a non negative integer -// */ -// const ZZ_LEXSTATE = [0, 0]; - -// /** -// * Translates characters to character classes -// */ -// const ZZ_CMAP_PACKED = -// "\11\0\1\1\1\1\1\0\1\1\1\1\22\0\1\1\7\0\1\0" + -// "\1\0\2\0\1\0\1\0\1\0\1\0\12\2\1\0\1\0\5\0" + -// "\32\3\1\0\1\0\1\0\1\0\1\0\1\0\32\3\1\0\1\0" + -// "\1\0\uff82\0"; - -// const ZZ_ACTION_PACKED_0 = "\1\0\1\1\1\2\1\3\1\4"; - -// const ZZ_ROWMAP_PACKED_0 = "\0\0\0\4\0\10\0\14\0\20"; - -// const ZZ_TRANS_PACKED_0 = "\1\2\1\3\1\4\1\5\5\0\1\3\4\0\1\4\4\0\1\5"; - -// const ZZ_ATTRIBUTE_PACKED_0 = "\1\0\1\11\3\1"; - -// /** -// * Translates characters to character classes -// */ -// const ZZ_CMAP = zzUnpackCMap(ZZ_CMAP_PACKED); - -// /** -// * Translates DFA states to action switch labels. -// */ -// const ZZ_ACTION = zzUnpackAction(); - -// function zzUnpackAction() { -// let result = Array(5); -// let offset = 0; -// offset = zzUnpackActionArgs(ZZ_ACTION_PACKED_0, offset, result); -// return result; -// } - -// function zzUnpackActionArgs(packed, offset, result) { -// let i = 0; /* index in packed string */ -// let j = offset; /* index in unpacked array */ -// let l = packed.length; -// while (i < l) { -// let count = packed.charAt(i++); -// let value = packed.charAt(i++); -// do { -// result[j++] = value; -// } while (--count > 0); -// } -// return j; -// } - -// /** -// * Translates a state to a row index in the transition table -// */ -// const ZZ_ROWMAP = zzUnpackRowMap(); - -// function zzUnpackRowMap() { -// let result = Array(5); -// let offset = 0; -// offset = zzUnpackRowMapArgs(ZZ_ROWMAP_PACKED_0, offset, result); -// return result; -// } - -// function zzUnpackRowMapArgs(packed, offset, result) { -// let i = 0; /* index in packed string */ -// let j = offset; /* index in unpacked array */ -// let l = packed.length; -// while (i < l) { -// let high = packed.charAt(i++) << 16; -// result[j++] = high | packed.charAt(i++); -// } -// return j; -// } - -// /** -// * The transition table of the DFA -// */ -// const ZZ_TRANS = zzUnpackTrans(); - -// function zzUnpackTrans() { -// let result = new Array(20); -// let offset = 0; -// offset = zzUnpackTransArgs(ZZ_TRANS_PACKED_0, offset, result); -// return result; -// } - -// function zzUnpackTransArgs(packed, offset, result) { -// let i = 0; /* index in packed string */ -// let j = offset; /* index in unpacked array */ -// let l = packed.length; -// while (i < l) { -// let count = packed.charAt(i++); -// let value = packed.charAt(i++); -// value--; -// do result[j++] = value; -// while (--count > 0); -// } -// return j; -// } - -// /* error codes */ -// const ZZ_UNKNOWN_ERROR = 0; -// const ZZ_NO_MATCH = 1; -// const ZZ_PUSHBACK_2BIG = 2; - -// /* error messages for the codes above */ -// const ZZ_ERROR_MSG = [ -// "Unkown internal scanner error", -// "Error: could not match input", -// "Error: pushback value was too large", -// ]; - -// /** -// * ZZ_ATTRIBUTE[aState] contains the attributes of state aState -// */ -// const ZZ_ATTRIBUTE = zzUnpackAttribute(); - -// function zzUnpackAttribute() { -// let result = Array(5); -// let offset = 0; -// offset = zzUnpackAttributeArgs(ZZ_ATTRIBUTE_PACKED_0, offset, result); -// return result; -// } - -// function zzUnpackAttributeArgs(packed, offset, result) { -// let i = 0; /* index in packed string */ -// let j = offset; /* index in unpacked array */ -// let l = packed.length; -// while (i < l) { -// let count = packed.charAt(i++); -// let value = packed.charAt(i++); -// do { -// result[j++] = value; -// } while (--count > 0); -// } -// return j; -// } - -// /** -// * Unpacks the compressed character translation table. -// * -// * @param packed the packed character translation table -// * @return the unpacked character translation table -// */ -// function zzUnpackCMap(packed) { -// let map = Array(0x10000); -// let i = 0; /* index in packed string */ -// let j = 0; /* index in unpacked array */ -// while (i < 64) { -// let count = packed.charAt(i++); -// let value = packed.charAt(i++); -// do { -// map[j++] = value; -// } while (--count > 0); -// } -// return map; -// } - -// class LexerOld { -// /** the input device */ -// zzReader; - -// /** the current state of the DFA */ -// zzState; - -// /** the current lexical state */ -// zzLexicalState = YYINITIAL; - -// /** -// * this buffer contains the current text to be matched and is -// * the source of the yytext() string -// */ -// zzBuffer = Array(ZZ_BUFFERSIZE); - -// /** the textposition at the last accepting state */ -// zzMarkedPos; - -// /** the current text position in the buffer */ -// zzCurrentPos; - -// /** startRead marks the beginning of the yytext() string in the buffer */ -// zzStartRead; - -// /** -// * endRead marks the last character in the buffer, that has been read -// * from input -// */ -// zzEndRead; - -// /** number of newlines encountered up to the start of the matched text */ -// yyline; - -// /** the number of characters up to the start of the matched text */ -// yychar; - -// /** -// * the number of characters from the last newline up to the start of the -// * matched text -// */ -// yycolumn; - -// /** -// * zzAtBOL == true <=> the scanner is currently at the beginning of a line -// */ -// zzAtBOL = true; - -// /** zzAtEOF == true <=> the scanner is at the EOF */ -// zzAtEOF; - -// /** denotes if the user-EOF-code has already been executed */ -// zzEOFDone; - -// constructor(reader) { -// this.zzReader = reader; -// } - -// scan() { -// let zzInput; -// let zzAction; - -// // cached fields: -// let zzCurrentPosL; -// let zzMarkedPosL; -// let zzEndReadL = this.zzEndRead; -// let zzBufferL = this.zzBuffer; -// let zzCMapL = ZZ_CMAP; - -// let zzTransL = ZZ_TRANS; -// let zzRowMapL = ZZ_ROWMAP; -// let zzAttrL = ZZ_ATTRIBUTE; - -// zzForAction: { -// while (true) { -// if (zzCurrentPosL < zzEndReadL) { -// zzInput = zzBufferL[zzCurrentPosL++]; -// } else if (this.zzAtEOF) { -// zzInput = YYEOF; -// break zzForAction; -// } else { -// // store back cached positions -// this.zzCurrentPos = zzCurrentPosL; -// this.zzMarkedPos = zzMarkedPosL; -// let eof = ""; -// // get translated positions and possibly new buffer -// zzCurrentPosL = this.zzCurrentPos; -// zzMarkedPosL = this.zzMarkedPos; -// zzBufferL = this.zzBuffer; -// zzEndReadL = this.zzEndRead; -// if (eof) { -// zzInput = YYEOF; -// break zzForAction; -// } else { -// zzInput = zzBufferL[zzCurrentPosL++]; -// } -// } -// let zzNext = zzTransL[zzRowMapL[this.zzState] + zzCMapL[zzInput]]; -// if (zzNext == -1) { -// break zzForAction; -// } -// this.zzState = zzNext; - -// let zzAttributes = zzAttrL[this.zzState]; -// if ((zzAttributes & 1) == 1) { -// zzAction = this.zzState; -// zzMarkedPosL = zzCurrentPosL; -// if ((zzAttributes & 8) == 8) { -// break zzForAction; -// } -// } - -// // store back cached position -// this.zzMarkedPos = zzMarkedPosL; - -// switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) { -// case 2: { -// return new TSpace(); -// } -// case 5: -// break; -// case 1: { -// return new TSymbol(yytext()); -// } -// case 6: -// break; -// case 4: { -// return new TWord(yytext()); -// } -// case 7: -// break; -// case 3: { -// return new TInt(yytext()); -// } -// case 8: -// break; -// default: -// if (zzInput == YYEOF && this.zzStartRead == this.zzCurrentPos) { -// this.zzAtEOF = true; -// return null; -// } else { -// zzScanError(ZZ_NO_MATCH); -// } -// } -// } -// } -// } -// } diff --git a/add-in/src/api/pids/archive/parsingTest.ts b/add-in/src/api/pids/archive/parsingTest.ts deleted file mode 100644 index 1f7b46b..0000000 --- a/add-in/src/api/pids/archive/parsingTest.ts +++ /dev/null @@ -1,90 +0,0 @@ - -const pattern = "(`(- )``;`=`IF`(``>` ``,``,` ``)`;`=`SUM`(``,` `B``)`;`=`C``+`)" -const ori_pat = "(,(-,),,;,=,IF,(,,>, ,,,,,,, ,,),;,=,SUM,(,,,, ,B,,),;,=,C,,+,)" - -const formMatrix = [ - ["-5", "=IF(A16 > 0, A16, 0)", "=SUM(A16, B16)", "=C16+1"], - ["-4", "=IF(A17 > 0, A17, 0)", "=SUM(A17, B17)", "=C17+1"], - ["-3", "=IF(A18 > 0, A18, 0)", "=SUM(A18, B18)", "=C18+1"], - ["-2", "=IF(A19 > 0, A19, 0)", "=SUM(A19, B19)", "=C19+1"], - ["-1", "=IF(A20 > 0, A20, 0)", "=SUM(A20, B20)", "=C20+1"], - ["0", "=IF(A21 > 0, A21, 0)", "=SUM(A21, B21)", "=C21+1"], - ["1", "=IF(A22 > 0, A22, 0)", "=SUM(A22, B22)", "=C22+1"], - ["2", "=IF(A23 > 0, A23, 0)", "=SUM(A23, B23)", "=C23+1"], - ["3", "=IF(A24 > 0, A24, 0)", "=SUM(A24, B24)", "=C24+1"], - ["4", "=IF(A25 > 0, A25, 0)", "=SUM(A25, B25)", "=C25+1"], - ["5", "=IF(A26 > 0, A26, 0)", "=SUM(A26, B26)", "=C26+1"], - ["6", "=IF(A27 > 0, A27, 0)", "=SUM(A27, B27)", "=C27+1"], - ["7", "=IF(A28 > 0, A28, 0)", "=SUM(A28, B28)", "=C28+1"], - ["8", "=IF(A29 > 0, A29, 0)", "=SUM(A29, B29)", "=C29+1"], - ["9", "=IF(A30 > 0, A30, 0)", "=SUM(A30, B30)", "=C30+1"], -] - -const typeSizes = { - "undefined": () => 0, - "boolean": () => 4, - "number": () => 8, - "string": item => 2 * item.length, - "object": item => !item ? 0 : Object - .keys(item) - .reduce((total, key) => sizeof(key) + sizeof(item[key]) + total, 0) - }; - -const sizeof = value => typeSizes[typeof value](value); - -console.log("Size of formula matrix:", sizeof(formMatrix)) - -function trim(s, c) { - if (c === "]") c = "\\]"; - if (c === "^") c = "\\^"; - if (c === "\\") c = "\\\\"; - return s.replace(new RegExp( - "^[" + c + "]+|[" + c + "]+$", "g" - ), ""); -} - -function isNumber(c) { - if (typeof c !== 'string') { - return false; - } - - if (c.trim() === '') { - return false; - } - - return false;//!isNaN(c); -} - -function compressFormulaMatrix(formMatrix, pattern) { - if (pattern.substring(0, 3) === "") { - pattern = pattern.substring(4, pattern.length - 1) - } - - let patterns = pattern.split(";") - let result = { 0: pattern } - - for (let i = 0; i < patterns.length; i += 1) { - patterns[i] = patterns[i].split("`") - patterns[i] = patterns[i].filter(s => { return s.length !== 0 }) - } - - for (let row = 0; row < formMatrix.length; row += 1) { - for (let col = 0; col < formMatrix[row].length; col += 1) { - let p = patterns[col] - let f = formMatrix[row][col] - for (let i = 0; i < p.length; i += 1) { - if (p[i].startsWith(", , 0, ,, , A, 16, ,, , 0, ), ;, =, SUM, (, A, 16, ,, , B, 16, ), ;, =, C, 16, +, 1) -ArrayBuffer(-, 4, ;, =, IF, (, A, 17, , >, , 0, ,, , A, 17, ,, , 0, ), ;, =, SUM, (, A, 17, ,, , B, 17, ), ;, =, C, 17, +, 1) -ArrayBuffer(-, 3, ;, =, IF, (, A, 18, , >, , 0, ,, , A, 18, ,, , 0, ), ;, =, SUM, (, A, 18, ,, , B, 18, ), ;, =, C, 18, +, 1) -ArrayBuffer(-, 2, ;, =, IF, (, A, 19, , >, , 0, ,, , A, 19, ,, , 0, ), ;, =, SUM, (, A, 19, ,, , B, 19, ), ;, =, C, 19, +, 1) -ArrayBuffer(-, 1, ;, =, IF, (, A, 20, , >, , 0, ,, , A, 20, ,, , 0, ), ;, =, SUM, (, A, 20, ,, , B, 20, ), ;, =, C, 20, +, 1) -ArrayBuffer(0, ;, =, IF, (, A, 21, , >, , 0, ,, , A, 21, ,, , 0, ), ;, =, SUM, (, A, 21, ,, , B, 21, ), ;, =, C, 21, +, 1) -ArrayBuffer(1, ;, =, IF, (, A, 22, , >, , 0, ,, , A, 22, ,, , 0, ), ;, =, SUM, (, A, 22, ,, , B, 22, ), ;, =, C, 22, +, 1) -ArrayBuffer(2, ;, =, IF, (, A, 23, , >, , 0, ,, , A, 23, ,, , 0, ), ;, =, SUM, (, A, 23, ,, , B, 23, ), ;, =, C, 23, +, 1) -ArrayBuffer(3, ;, =, IF, (, A, 24, , >, , 0, ,, , A, 24, ,, , 0, ), ;, =, SUM, (, A, 24, ,, , B, 24, ), ;, =, C, 24, +, 1) -ArrayBuffer(4, ;, =, IF, (, A, 25, , >, , 0, ,, , A, 25, ,, , 0, ), ;, =, SUM, (, A, 25, ,, , B, 25, ), ;, =, C, 25, +, 1) -ArrayBuffer(5, ;, =, IF, (, A, 26, , >, , 0, ,, , A, 26, ,, , 0, ), ;, =, SUM, (, A, 26, ,, , B, 26, ), ;, =, C, 26, +, 1) -ArrayBuffer(6, ;, =, IF, (, A, 27, , >, , 0, ,, , A, 27, ,, , 0, ), ;, =, SUM, (, A, 27, ,, , B, 27, ), ;, =, C, 27, +, 1) -ArrayBuffer(7, ;, =, IF, (, A, 28, , >, , 0, ,, , A, 28, ,, , 0, ), ;, =, SUM, (, A, 28, ,, , B, 28, ), ;, =, C, 28, +, 1) -ArrayBuffer(8, ;, =, IF, (, A, 29, , >, , 0, ,, , A, 29, ,, , 0, ), ;, =, SUM, (, A, 29, ,, , B, 29, ), ;, =, C, 29, +, 1) -ArrayBuffer(9, ;, =, IF, (, A, 30, , >, , 0, ,, , A, 30, ,, , 0, ), ;, =, SUM, (, A, 30, ,, , B, 30, ), ;, =, C, 30, +, 1) \ No newline at end of file From cd8693e82b33f948846ef3119fe8353be17d03b8 Mon Sep 17 00:00:00 2001 From: Connor Lien Date: Mon, 5 Sep 2022 16:23:35 -0700 Subject: [PATCH 5/9] Fix tail case for commonSymbolRule --- .gitignore | 1 + add-in/src/api/pids/pattern/patternMiner.ts | 31 +- add-in/src/api/pids/rule/commonSymbolRule.ts | 22 +- .../test_notebook-checkpoint.ipynb | 508 ++++++++++++++++++ add-in/src/api/pids/test/minerTest.ts | 8 +- add-in/src/api/pids/test/test.json | 452 ++++++++++++++++ add-in/src/api/pids/test/testMatrix.ts | 32 +- add-in/src/api/pids/test/test_address.csv | 10 + add-in/src/api/pids/test/test_ip.csv | 10 + add-in/src/api/pids/test/test_notebook.ipynb | 84 +++ add-in/src/api/pids/test/tokenizeTest.ts | 2 +- add-in/src/api/pids/utils/file.ts | 18 + 12 files changed, 1157 insertions(+), 21 deletions(-) create mode 100644 add-in/src/api/pids/test/.ipynb_checkpoints/test_notebook-checkpoint.ipynb create mode 100644 add-in/src/api/pids/test/test.json create mode 100644 add-in/src/api/pids/test/test_address.csv create mode 100644 add-in/src/api/pids/test/test_ip.csv create mode 100644 add-in/src/api/pids/test/test_notebook.ipynb create mode 100644 add-in/src/api/pids/utils/file.ts diff --git a/.gitignore b/.gitignore index 7fd525e..fcd85a2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,6 @@ node_modules .vscode target .idea +.DS_Store /add-in/dist diff --git a/add-in/src/api/pids/pattern/patternMiner.ts b/add-in/src/api/pids/pattern/patternMiner.ts index 2cb3a06..14b7ff1 100644 --- a/add-in/src/api/pids/pattern/patternMiner.ts +++ b/add-in/src/api/pids/pattern/patternMiner.ts @@ -24,7 +24,7 @@ export class PatternMiner { refineResult = refined[0]; } } - return refineResult; + return this.repack(refineResult, lines.length); } refine(root) { @@ -39,4 +39,33 @@ export class PatternMiner { } return [current, null]; } + + repack(splitArr, entries) { + let result = []; + let getPSeqString = (seq) => seq.content.map((token) => token.toString()).join(""); + for (let idx = 0; idx < entries; idx += 1) { + let line = []; + let str: string; + splitArr.forEach((item) => { + if (item instanceof PUnion) { + let val = item.content[idx]; + if (val instanceof PSeq) { + str = getPSeqString(val); + } else if (val instanceof PToken) { + str = val.token.toString(); + } + } else if (item instanceof PSeq) { + str = getPSeqString(item); + } else if (item instanceof PToken) { + str = item.token.toString(); + } else { + str = item.toString(); + } + line.push(str); + }); + result.push(line); + } + + return result; + } } diff --git a/add-in/src/api/pids/rule/commonSymbolRule.ts b/add-in/src/api/pids/rule/commonSymbolRule.ts index 6f84f52..1f6b50b 100644 --- a/add-in/src/api/pids/rule/commonSymbolRule.ts +++ b/add-in/src/api/pids/rule/commonSymbolRule.ts @@ -2,7 +2,6 @@ import { RewriteRule } from "./rewriteRule"; import { PUnion, PSeq, PToken, PEmpty } from "../pattern/pattern"; import { TSymbol } from "../tokenize/token"; import { CommonSeq } from "./commonSeq"; -import { stringifyArray, stringifyMatrix } from "../utils/stringify"; export class CommonSymbolRule extends RewriteRule { condition(ptn) { @@ -57,6 +56,8 @@ export class CommonSymbolRule extends RewriteRule { }) ); + console.log(symbolsWithPos); + // Valid lines have at least one symbol let noSymbolLines = new Set(); let validLinesWithIndex = new Set>(); @@ -119,6 +120,25 @@ export class CommonSymbolRule extends RewriteRule { resultSeq.push(symbol); }); + let afterSymbolUnion = []; + let nonEmpty = false; + commonSymbolsWithPos.forEach((line, lineIdx) => { + let startIdx = line[line.length - 1][1] + 1; + const unionLine: any[] = union.content[lineIdx].content; + let endIdx = unionLine.length; + + if (endIdx - startIdx > 0) { + let unit = unionLine.slice(startIdx, endIdx); + afterSymbolUnion.push(new PSeq(unit)); + nonEmpty = true; + } else { + afterSymbolUnion.push(new PEmpty()); + } + }); + if (nonEmpty) { + resultSeq.push(new PUnion(afterSymbolUnion)); + } + return new PSeq(resultSeq); } } diff --git a/add-in/src/api/pids/test/.ipynb_checkpoints/test_notebook-checkpoint.ipynb b/add-in/src/api/pids/test/.ipynb_checkpoints/test_notebook-checkpoint.ipynb new file mode 100644 index 0000000..97b6708 --- /dev/null +++ b/add-in/src/api/pids/test/.ipynb_checkpoints/test_notebook-checkpoint.ipynb @@ -0,0 +1,508 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "d7b128cc", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import lux" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "67caa419", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
0123456789...17181920212223242526
0-5=IF(A1>0,...SUM(A1,B1)=C1+
1-4=IF(A2>0,...SUM(A2,B2)=C2+
2-3=IF(A3>0,...SUM(A3,B3)=C3+
3-2=IF(A4>0,...SUM(A4,B4)=C4+
4-1=IF(A5>0,...SUM(A5,B5)=C5+
50=IF(A6>0,...SUM(A6,B6)=C6+
61=IF(A7>0,...SUM(A7,B7)=C7+
72=IF(A8>0,...SUM(A8,B8)=C8+
83=IF(A9>0,...SUM(A9,B9)=C9+
94=IF(A10>0,...SUM(A10,B10)=C10+
105=IF(A11>0,...SUM(A11,B11)=C11+
116=IF(A12>0,...SUM(A12,B12)=C12+
127=IF(A13>0,...SUM(A13,B13)=C13+
138=IF(A14>0,...SUM(A14,B14)=C14+
149=IF(A15>0,...SUM(A15,B15)=C15+
\n", + "

15 rows × 27 columns

\n", + "
" + ], + "text/plain": [ + " 0 1 2 3 4 5 6 7 8 9 ... 17 18 19 20 21 22 23 24 \\\n", + "0 -5 = IF ( A1 > 0 , ... SUM ( A1 , B1 ) = \n", + "1 -4 = IF ( A2 > 0 , ... SUM ( A2 , B2 ) = \n", + "2 -3 = IF ( A3 > 0 , ... SUM ( A3 , B3 ) = \n", + "3 -2 = IF ( A4 > 0 , ... SUM ( A4 , B4 ) = \n", + "4 -1 = IF ( A5 > 0 , ... SUM ( A5 , B5 ) = \n", + "5 0 = IF ( A6 > 0 , ... SUM ( A6 , B6 ) = \n", + "6 1 = IF ( A7 > 0 , ... SUM ( A7 , B7 ) = \n", + "7 2 = IF ( A8 > 0 , ... SUM ( A8 , B8 ) = \n", + "8 3 = IF ( A9 > 0 , ... SUM ( A9 , B9 ) = \n", + "9 4 = IF ( A10 > 0 , ... SUM ( A10 , B10 ) = \n", + "10 5 = IF ( A11 > 0 , ... SUM ( A11 , B11 ) = \n", + "11 6 = IF ( A12 > 0 , ... SUM ( A12 , B12 ) = \n", + "12 7 = IF ( A13 > 0 , ... SUM ( A13 , B13 ) = \n", + "13 8 = IF ( A14 > 0 , ... SUM ( A14 , B14 ) = \n", + "14 9 = IF ( A15 > 0 , ... SUM ( A15 , B15 ) = \n", + "\n", + " 25 26 \n", + "0 C1 + \n", + "1 C2 + \n", + "2 C3 + \n", + "3 C4 + \n", + "4 C5 + \n", + "5 C6 + \n", + "6 C7 + \n", + "7 C8 + \n", + "8 C9 + \n", + "9 C10 + \n", + "10 C11 + \n", + "11 C12 + \n", + "12 C13 + \n", + "13 C14 + \n", + "14 C15 + \n", + "\n", + "[15 rows x 27 columns]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = pd.read_json(\"test.json\")\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "09faa345", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/add-in/src/api/pids/test/minerTest.ts b/add-in/src/api/pids/test/minerTest.ts index 6c2c8a2..e2b3d91 100644 --- a/add-in/src/api/pids/test/minerTest.ts +++ b/add-in/src/api/pids/test/minerTest.ts @@ -1,5 +1,9 @@ -import { testFormMatrix1 } from "./testMatrix"; +import { testFormMatrix } from "./testMatrix"; import { PatternMiner } from "../pattern/patternMiner"; +import { readFile, writeArray } from "../utils/file"; let miner = new PatternMiner(); -console.log(miner.mine(testFormMatrix1).toString()); +let data = testFormMatrix; // readFile("test_ip.csv"); +let arr = miner.mine(data); +console.log(arr); +writeArray(arr, "test.json"); diff --git a/add-in/src/api/pids/test/test.json b/add-in/src/api/pids/test/test.json new file mode 100644 index 0000000..2421c98 --- /dev/null +++ b/add-in/src/api/pids/test/test.json @@ -0,0 +1,452 @@ +[ + [ + "-5", + "=", + "IF", + "(", + "A1", + " ", + ">", + " ", + "0", + ",", + " ", + "A1", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A1", + ",", + " ", + "B1", + ")", + "=", + "C1", + "+", + "1" + ], + [ + "-4", + "=", + "IF", + "(", + "A2", + " ", + ">", + " ", + "0", + ",", + " ", + "A2", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A2", + ",", + " ", + "B2", + ")", + "=", + "C2", + "+", + "1" + ], + [ + "-3", + "=", + "IF", + "(", + "A3", + " ", + ">", + " ", + "0", + ",", + " ", + "A3", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A3", + ",", + " ", + "B3", + ")", + "=", + "C3", + "+", + "1" + ], + [ + "-2", + "=", + "IF", + "(", + "A4", + " ", + ">", + " ", + "0", + ",", + " ", + "A4", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A4", + ",", + " ", + "B4", + ")", + "=", + "C4", + "+", + "1" + ], + [ + "-1", + "=", + "IF", + "(", + "A5", + " ", + ">", + " ", + "0", + ",", + " ", + "A5", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A5", + ",", + " ", + "B5", + ")", + "=", + "C5", + "+", + "1" + ], + [ + "0", + "=", + "IF", + "(", + "A6", + " ", + ">", + " ", + "0", + ",", + " ", + "A6", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A6", + ",", + " ", + "B6", + ")", + "=", + "C6", + "+", + "1" + ], + [ + "1", + "=", + "IF", + "(", + "A7", + " ", + ">", + " ", + "0", + ",", + " ", + "A7", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A7", + ",", + " ", + "B7", + ")", + "=", + "C7", + "+", + "1" + ], + [ + "2", + "=", + "IF", + "(", + "A8", + " ", + ">", + " ", + "0", + ",", + " ", + "A8", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A8", + ",", + " ", + "B8", + ")", + "=", + "C8", + "+", + "1" + ], + [ + "3", + "=", + "IF", + "(", + "A9", + " ", + ">", + " ", + "0", + ",", + " ", + "A9", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A9", + ",", + " ", + "B9", + ")", + "=", + "C9", + "+", + "1" + ], + [ + "4", + "=", + "IF", + "(", + "A10", + " ", + ">", + " ", + "0", + ",", + " ", + "A10", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A10", + ",", + " ", + "B10", + ")", + "=", + "C10", + "+", + "1" + ], + [ + "5", + "=", + "IF", + "(", + "A11", + " ", + ">", + " ", + "0", + ",", + " ", + "A11", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A11", + ",", + " ", + "B11", + ")", + "=", + "C11", + "+", + "1" + ], + [ + "6", + "=", + "IF", + "(", + "A12", + " ", + ">", + " ", + "0", + ",", + " ", + "A12", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A12", + ",", + " ", + "B12", + ")", + "=", + "C12", + "+", + "1" + ], + [ + "7", + "=", + "IF", + "(", + "A13", + " ", + ">", + " ", + "0", + ",", + " ", + "A13", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A13", + ",", + " ", + "B13", + ")", + "=", + "C13", + "+", + "1" + ], + [ + "8", + "=", + "IF", + "(", + "A14", + " ", + ">", + " ", + "0", + ",", + " ", + "A14", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A14", + ",", + " ", + "B14", + ")", + "=", + "C14", + "+", + "1" + ], + [ + "9", + "=", + "IF", + "(", + "A15", + " ", + ">", + " ", + "0", + ",", + " ", + "A15", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A15", + ",", + " ", + "B15", + ")", + "=", + "C15", + "+", + "1" + ] +] \ No newline at end of file diff --git a/add-in/src/api/pids/test/testMatrix.ts b/add-in/src/api/pids/test/testMatrix.ts index b62eb17..76eff77 100644 --- a/add-in/src/api/pids/test/testMatrix.ts +++ b/add-in/src/api/pids/test/testMatrix.ts @@ -1,17 +1,17 @@ -export const testFormMatrix1 = [ - ["-5", "=IF(A16 > 0, A16, 0)", "=SUM(A16, B16)", "=C16+1"], - ["-4", "=IF(A17 > 0, A17, 0)", "=SUM(A17, B17)", "=C17+1"], - ["-3", "=IF(A18 > 0, A18, 0)", "=SUM(A18, B18)", "=C18+1"], - ["-2", "=IF(A19 > 0, A19, 0)", "=SUM(A19, B19)", "=C19+1"], - ["-1", "=IF(A20 > 0, A20, 0)", "=SUM(A20, B20)", "=C20+1"], - ["0", "=IF(A21 > 0, A21, 0)", "=SUM(A21, B21)", "=C21+1"], - ["1", "=IF(A22 > 0, A22, 0)", "=SUM(A22, B22)", "=C22+1"], - ["2", "=IF(A23 > 0, A23, 0)", "=SUM(A23, B23)", "=C23+1"], - ["3", "=IF(A24 > 0, A24, 0)", "=SUM(A24, B24)", "=C24+1"], - ["4", "=IF(A25 > 0, A25, 0)", "=SUM(A25, B25)", "=C25+1"], - ["5", "=IF(A26 > 0, A26, 0)", "=SUM(A26, B26)", "=C26+1"], - ["6", "=IF(A27 > 0, A27, 0)", "=SUM(A27, B27)", "=C27+1"], - ["7", "=IF(A28 > 0, A28, 0)", "=SUM(A28, B28)", "=C28+1"], - ["8", "=IF(A29 > 0, A29, 0)", "=SUM(A29, B29)", "=C29+1"], - ["9", "=IF(A30 > 0, A30, 0)", "=SUM(A30, B30)", "=C30+1"], +export const testFormMatrix = [ + ["-5", "=IF(A1 > 0, A1, 0)", "=SUM(A1, B1)", "=C1+1"], + ["-4", "=IF(A2 > 0, A2, 0)", "=SUM(A2, B2)", "=C2+1"], + ["-3", "=IF(A3 > 0, A3, 0)", "=SUM(A3, B3)", "=C3+1"], + ["-2", "=IF(A4 > 0, A4, 0)", "=SUM(A4, B4)", "=C4+1"], + ["-1", "=IF(A5 > 0, A5, 0)", "=SUM(A5, B5)", "=C5+1"], + ["0", "=IF(A6 > 0, A6, 0)", "=SUM(A6, B6)", "=C6+1"], + ["1", "=IF(A7 > 0, A7, 0)", "=SUM(A7, B7)", "=C7+1"], + ["2", "=IF(A8 > 0, A8, 0)", "=SUM(A8, B8)", "=C8+1"], + ["3", "=IF(A9 > 0, A9, 0)", "=SUM(A9, B9)", "=C9+1"], + ["4", "=IF(A10 > 0, A10, 0)", "=SUM(A10, B10)", "=C10+1"], + ["5", "=IF(A11 > 0, A11, 0)", "=SUM(A11, B11)", "=C11+1"], + ["6", "=IF(A12 > 0, A12, 0)", "=SUM(A12, B12)", "=C12+1"], + ["7", "=IF(A13 > 0, A13, 0)", "=SUM(A13, B13)", "=C13+1"], + ["8", "=IF(A14 > 0, A14, 0)", "=SUM(A14, B14)", "=C14+1"], + ["9", "=IF(A15 > 0, A15, 0)", "=SUM(A15, B15)", "=C15+1"], ]; diff --git a/add-in/src/api/pids/test/test_address.csv b/add-in/src/api/pids/test/test_address.csv new file mode 100644 index 0000000..1d6b7c4 --- /dev/null +++ b/add-in/src/api/pids/test/test_address.csv @@ -0,0 +1,10 @@ +388,1st Hillcrest,RD,Suite 480,Lakeland,Weber County,WI,90894 +907,Highland South,Wy,Suite V,Denver,Oconto County,NH,05125 +355,13th Davis,Parkway,Suite R,Oak Hill,Duplin County,OK,63524 +573,Davis West,Cir.,Suite G,Freeman,Crane County,NM,66909 +543,Seventh Central,Ct.,Suite 130,Carlton,Pendleton County,WV,48722 +940,Church Maple,Avenue,Suite 320,Hazelwood,Coos County,SD,43957 +775,Franklin Smith,Road,Suite 10,Sanford,Searcy County,GA,50725 +536,Main 10th,Ave,Suite 400,Brandon,Evangeline Parish,NY,75274 +639,Eigth Cherry,Boulevard,Suite H,White Oak,Anne Arundel County,UT,85819 +798,Hillcrest East,Drive,Suite 90,Blue Springs,Salt Lake County,UT,98579 \ No newline at end of file diff --git a/add-in/src/api/pids/test/test_ip.csv b/add-in/src/api/pids/test/test_ip.csv new file mode 100644 index 0000000..cb35a84 --- /dev/null +++ b/add-in/src/api/pids/test/test_ip.csv @@ -0,0 +1,10 @@ +9755:FE45:E943:6B7F:75D6:8BC5:F0AD:BCBC +5464:F26A:4EC5:5559:43D0:9710:859F:AD3D +DF4F:61CB:1A3A:721D:B0C2:04AD:CDA9:1188 +7DAE:0162:C620:716C:C25E:6E58:CEB8:194C +759C:1B54:42CB:C5A2:DD0A:6F3C:76B4:A3BD +4222:B2D9:5769:00F3:9C20:F83F:67C0:469B +89BA:F2F0:2CD9:320C:3DAD:A69E:16B6:6D00 +A69D:046D:8FD4:2E2F:B111:C305:6619:8734 +C8ED:72D8:54F0:F9AD:4A75:A09F:D179:9542 +A3A3:DCC2:C369:886A:A2C3:F5F8:652F:09B5 \ No newline at end of file diff --git a/add-in/src/api/pids/test/test_notebook.ipynb b/add-in/src/api/pids/test/test_notebook.ipynb new file mode 100644 index 0000000..099c917 --- /dev/null +++ b/add-in/src/api/pids/test/test_notebook.ipynb @@ -0,0 +1,84 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "6b3312fb", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import lux" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "6792fb2c", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b88a86b2b6944557b1271c33658a8c10", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Button(description='Toggle Pandas/Lux', layout=Layout(top='5px', width='140px'), style=ButtonStyle())" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c14981a079c04fb59a7fbd967b82db8b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "df = pd.read_json(\"test.json\")\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3dc21294", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/add-in/src/api/pids/test/tokenizeTest.ts b/add-in/src/api/pids/test/tokenizeTest.ts index eee3294..c641a10 100644 --- a/add-in/src/api/pids/test/tokenizeTest.ts +++ b/add-in/src/api/pids/test/tokenizeTest.ts @@ -1,4 +1,4 @@ -import { testFormMatrix } from "./testMatrix"; +import { testFormMatrix1 } from "./testMatrix"; import { stringifyArray } from "../utils/stringify"; import { Tokenizer } from "../tokenize/tokenizer"; diff --git a/add-in/src/api/pids/utils/file.ts b/add-in/src/api/pids/utils/file.ts new file mode 100644 index 0000000..b7c0438 --- /dev/null +++ b/add-in/src/api/pids/utils/file.ts @@ -0,0 +1,18 @@ +let fs = require("fs"); +const path = "add-in/src/api/pids/test/"; + +export function readFile(filename: string) { + let text = fs.readFileSync(path + filename, "utf-8"); + let data = text.split("\n").map((row) => row.split(",")); + return data; +} + +export function writeArray(arr: any[], filename: string) { + let payload = JSON.stringify(arr, null, 2); + let err = (err) => { + if (err) { + console.log(err); + } + }; + fs.writeFile(path + filename, payload, err); +} From 866e2782466c5515e13e824765b22438e44f5d8b Mon Sep 17 00:00:00 2001 From: Connor Lien Date: Mon, 5 Sep 2022 18:52:25 -0700 Subject: [PATCH 6/9] CommonWordRule --- add-in/src/api/pids/pattern/patternMiner.ts | 58 ++- add-in/src/api/pids/rule/commonSymbolRule.ts | 2 - add-in/src/api/pids/rule/commonWordRule.ts | 142 ++++++ .../test_notebook-checkpoint.ipynb | 302 ++++++----- add-in/src/api/pids/test/test.json | 225 ++++++--- add-in/src/api/pids/test/test_notebook.ipynb | 474 ++++++++++++++++-- 6 files changed, 919 insertions(+), 284 deletions(-) create mode 100644 add-in/src/api/pids/rule/commonWordRule.ts diff --git a/add-in/src/api/pids/pattern/patternMiner.ts b/add-in/src/api/pids/pattern/patternMiner.ts index 14b7ff1..d35f754 100644 --- a/add-in/src/api/pids/pattern/patternMiner.ts +++ b/add-in/src/api/pids/pattern/patternMiner.ts @@ -1,12 +1,15 @@ import { Tokenizer } from "../tokenize/tokenizer"; import { Pattern, PUnion, PSeq, PToken } from "./pattern"; import { CommonSymbolRule } from "../rule/commonSymbolRule"; +import { CommonWordRule } from "../rule/commonWordRule"; import { SameItemRule } from "../rule/sameItemRule"; export class PatternMiner { sampleSize = 500; - rules = [new CommonSymbolRule(), new SameItemRule()]; + rules = [new CommonSymbolRule(), new CommonWordRule(), new SameItemRule()]; + + getPSeqString = (seq) => seq.content.map((token) => token.toString()).join(""); mine(lines) { let tokens = lines.map((line) => Tokenizer.tokenize(line)); @@ -40,32 +43,41 @@ export class PatternMiner { return [current, null]; } - repack(splitArr, entries) { + repack(pattern, entries) { + if (!(pattern instanceof PSeq)) { + pattern = new PSeq(pattern); + } let result = []; - let getPSeqString = (seq) => seq.content.map((token) => token.toString()).join(""); for (let idx = 0; idx < entries; idx += 1) { + result.push(this.getPatternArray(pattern, idx)); + } + return result; + } + + getPatternArray(pattern, idx) { + if (pattern instanceof PToken) { + return [pattern.token.toString()]; + } + let hasSubpattern = false; + pattern.content.forEach((subpattern) => { + if (subpattern instanceof PSeq || subpattern instanceof PUnion) { + hasSubpattern = true; + } + }); + if (hasSubpattern && pattern instanceof PSeq) { let line = []; - let str: string; - splitArr.forEach((item) => { - if (item instanceof PUnion) { - let val = item.content[idx]; - if (val instanceof PSeq) { - str = getPSeqString(val); - } else if (val instanceof PToken) { - str = val.token.toString(); - } - } else if (item instanceof PSeq) { - str = getPSeqString(item); - } else if (item instanceof PToken) { - str = item.token.toString(); - } else { - str = item.toString(); - } - line.push(str); + pattern.content.forEach((subpattern) => { + let arr = this.getPatternArray(subpattern, idx); + line.push(...arr); }); - result.push(line); + return line; + } else if (hasSubpattern && pattern instanceof PUnion) { + return this.getPatternArray(pattern.content[idx], idx); + } else if (pattern instanceof PSeq) { + return [this.getPSeqString(pattern)]; + } else if (pattern instanceof PUnion) { + return [pattern.content[idx]]; } - - return result; + return []; } } diff --git a/add-in/src/api/pids/rule/commonSymbolRule.ts b/add-in/src/api/pids/rule/commonSymbolRule.ts index 1f6b50b..4c8cfd1 100644 --- a/add-in/src/api/pids/rule/commonSymbolRule.ts +++ b/add-in/src/api/pids/rule/commonSymbolRule.ts @@ -56,8 +56,6 @@ export class CommonSymbolRule extends RewriteRule { }) ); - console.log(symbolsWithPos); - // Valid lines have at least one symbol let noSymbolLines = new Set(); let validLinesWithIndex = new Set>(); diff --git a/add-in/src/api/pids/rule/commonWordRule.ts b/add-in/src/api/pids/rule/commonWordRule.ts new file mode 100644 index 0000000..51c364e --- /dev/null +++ b/add-in/src/api/pids/rule/commonWordRule.ts @@ -0,0 +1,142 @@ +import { RewriteRule } from "./rewriteRule"; +import { PUnion, PSeq, PToken, PEmpty } from "../pattern/pattern"; +import { TWord } from "../tokenize/token"; +import { CommonSeq } from "./commonSeq"; + +export class CommonWordRule extends RewriteRule { + condition(ptn) { + if (!(ptn instanceof PUnion)) { + return false; + } + + let cnt = ptn.content; + if (cnt.length <= 1) { + return false; + } + + let res = cnt.map((x) => { + if (x instanceof PSeq) { + return [x.content.every((t) => t instanceof PToken || t == PEmpty), x.content.length]; + } else if (x instanceof PToken) { + return [true, 1]; + } else if (x == PEmpty) { + return [true, 1]; + } else { + return [false, 0]; + } + }); + + return res.some((x) => x[1] > 1) && res.every((x) => x[0]); + } + + update(union: PUnion) { + // Flatten the union content + let unionData = union.content.map((item) => { + let result = [item]; + if (item instanceof PSeq) { + result = item.content; + } else if (item instanceof PEmpty) { + result = []; + } + return result; + }); + if (unionData.length === 1) { + return union; + } + + // Scan union data for symbols + let symbolsWithPos = unionData.map((line) => + line + .map((value, index) => [value, index]) + .filter((value) => { + if (value[0] instanceof PToken) { + return value[0].token instanceof TWord; + } + return false; + }) + ); + + // Valid lines have at least one symbol + let noSymbolLines = new Set(); + let validLinesWithIndex = new Set>(); + symbolsWithPos + .map((value, idx) => [value, idx]) + .forEach((value) => { + if ((value[0] as Array).length === 0) { + noSymbolLines.add(value[1] as number); + } else { + validLinesWithIndex.add(value); + } + }); + + if (noSymbolLines.size > 0) { + return union; + } + + let validLines = Array.from(validLinesWithIndex).map((value) => value[0]); + + // Determine common symbols and match symbol in each line to the common + let commonSeq = new CommonSeq(); + const eq = (a: Array, b: Array) => (a[0] as PToken).toString() === (b[0] as PToken).toString(); + let commonSymbols = commonSeq.find(validLines, eq).map((i) => i[0]); + + // Filter out symbols that are not common + let commonSymbolsWithPos = []; + symbolsWithPos.forEach((line) => { + let idx = 0; + let commonSymbolLine = []; + commonSymbols.forEach((symbol) => { + while (line[idx] && symbol.token.value !== line[idx][0].token.value) { + idx += 1; + } + commonSymbolLine.push(line[idx]); + }); + commonSymbolsWithPos.push(commonSymbolLine); + }); + + // Package new union split by common symbols + let resultSeq = []; + this.happen(); + commonSymbols.forEach((symbol, symbolIdx) => { + let beforeSymbolUnion = []; + let nonEmpty: boolean = false; + commonSymbolsWithPos.forEach((line, lineIdx) => { + let startIdx = symbolIdx == 0 ? 0 : line[symbolIdx - 1][1] + 1; + let endIdx = line[symbolIdx][1]; + const unionLine: any[] = union.content[lineIdx].content; + if (endIdx - startIdx > 0) { + let unit = unionLine.slice(startIdx, endIdx); + beforeSymbolUnion.push(new PSeq(unit)); + nonEmpty = true; + } else { + beforeSymbolUnion.push(new PEmpty()); + } + }); + if (nonEmpty) { + resultSeq.push(new PUnion(beforeSymbolUnion)); + } + resultSeq.push(symbol); + }); + + let afterSymbolUnion = []; + let nonEmpty = false; + commonSymbolsWithPos.forEach((line, lineIdx) => { + let startIdx = line[line.length - 1][1] + 1; + const unionLine: any[] = union.content[lineIdx].content; + let endIdx = unionLine.length; + + if (endIdx - startIdx > 0) { + let unit = unionLine.slice(startIdx, endIdx); + afterSymbolUnion.push(new PSeq(unit)); + nonEmpty = true; + } else { + afterSymbolUnion.push(new PEmpty()); + } + }); + if (nonEmpty) { + resultSeq.push(new PUnion(afterSymbolUnion)); + } + + return new PSeq(resultSeq); + } +} diff --git a/add-in/src/api/pids/test/.ipynb_checkpoints/test_notebook-checkpoint.ipynb b/add-in/src/api/pids/test/.ipynb_checkpoints/test_notebook-checkpoint.ipynb index 97b6708..c1b3750 100644 --- a/add-in/src/api/pids/test/.ipynb_checkpoints/test_notebook-checkpoint.ipynb +++ b/add-in/src/api/pids/test/.ipynb_checkpoints/test_notebook-checkpoint.ipynb @@ -2,10 +2,21 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, - "id": "d7b128cc", + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'lux'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mlux\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'lux'" + ] + } + ], "source": [ "import pandas as pd\n", "import lux" @@ -13,8 +24,7 @@ }, { "cell_type": "code", - "execution_count": 4, - "id": "67caa419", + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -49,16 +59,16 @@ " 8\n", " 9\n", " ...\n", - " 17\n", - " 18\n", - " 19\n", - " 20\n", - " 21\n", - " 22\n", " 23\n", " 24\n", " 25\n", " 26\n", + " 27\n", + " 28\n", + " 29\n", + " 30\n", + " 31\n", + " 32\n", " \n", " \n", " \n", @@ -68,23 +78,23 @@ " =\n", " IF\n", " (\n", - " A1\n", + " A\n", + " 1\n", " \n", " >\n", " \n", " 0\n", - " ,\n", " ...\n", - " SUM\n", - " (\n", - " A1\n", " ,\n", " \n", - " B1\n", + " B\n", + " 1\n", " )\n", " =\n", - " C1\n", + " C\n", + " 1\n", " +\n", + " 1\n", " \n", " \n", " 1\n", @@ -92,23 +102,23 @@ " =\n", " IF\n", " (\n", - " A2\n", + " A\n", + " 2\n", " \n", " >\n", " \n", " 0\n", - " ,\n", " ...\n", - " SUM\n", - " (\n", - " A2\n", " ,\n", " \n", - " B2\n", + " B\n", + " 2\n", " )\n", " =\n", - " C2\n", + " C\n", + " 2\n", " +\n", + " 1\n", " \n", " \n", " 2\n", @@ -116,23 +126,23 @@ " =\n", " IF\n", " (\n", - " A3\n", + " A\n", + " 3\n", " \n", " >\n", " \n", " 0\n", - " ,\n", " ...\n", - " SUM\n", - " (\n", - " A3\n", " ,\n", " \n", - " B3\n", + " B\n", + " 3\n", " )\n", " =\n", - " C3\n", + " C\n", + " 3\n", " +\n", + " 1\n", " \n", " \n", " 3\n", @@ -140,23 +150,23 @@ " =\n", " IF\n", " (\n", - " A4\n", + " A\n", + " 4\n", " \n", " >\n", " \n", " 0\n", - " ,\n", " ...\n", - " SUM\n", - " (\n", - " A4\n", " ,\n", " \n", - " B4\n", + " B\n", + " 4\n", " )\n", " =\n", - " C4\n", + " C\n", + " 4\n", " +\n", + " 1\n", " \n", " \n", " 4\n", @@ -164,23 +174,23 @@ " =\n", " IF\n", " (\n", - " A5\n", + " A\n", + " 5\n", " \n", " >\n", " \n", " 0\n", - " ,\n", " ...\n", - " SUM\n", - " (\n", - " A5\n", " ,\n", " \n", - " B5\n", + " B\n", + " 5\n", " )\n", " =\n", - " C5\n", + " C\n", + " 5\n", " +\n", + " 1\n", " \n", " \n", " 5\n", @@ -188,23 +198,23 @@ " =\n", " IF\n", " (\n", - " A6\n", + " A\n", + " 6\n", " \n", " >\n", " \n", " 0\n", - " ,\n", " ...\n", - " SUM\n", - " (\n", - " A6\n", " ,\n", " \n", - " B6\n", + " B\n", + " 6\n", " )\n", " =\n", - " C6\n", + " C\n", + " 6\n", " +\n", + " 1\n", " \n", " \n", " 6\n", @@ -212,23 +222,23 @@ " =\n", " IF\n", " (\n", - " A7\n", + " A\n", + " 7\n", " \n", " >\n", " \n", " 0\n", - " ,\n", " ...\n", - " SUM\n", - " (\n", - " A7\n", " ,\n", " \n", - " B7\n", + " B\n", + " 7\n", " )\n", " =\n", - " C7\n", + " C\n", + " 7\n", " +\n", + " 1\n", " \n", " \n", " 7\n", @@ -236,23 +246,23 @@ " =\n", " IF\n", " (\n", - " A8\n", + " A\n", + " 8\n", " \n", " >\n", " \n", " 0\n", - " ,\n", " ...\n", - " SUM\n", - " (\n", - " A8\n", " ,\n", " \n", - " B8\n", + " B\n", + " 8\n", " )\n", " =\n", - " C8\n", + " C\n", + " 8\n", " +\n", + " 1\n", " \n", " \n", " 8\n", @@ -260,23 +270,23 @@ " =\n", " IF\n", " (\n", - " A9\n", + " A\n", + " 9\n", " \n", " >\n", " \n", " 0\n", - " ,\n", " ...\n", - " SUM\n", - " (\n", - " A9\n", " ,\n", " \n", - " B9\n", + " B\n", + " 9\n", " )\n", " =\n", - " C9\n", + " C\n", + " 9\n", " +\n", + " 1\n", " \n", " \n", " 9\n", @@ -284,23 +294,23 @@ " =\n", " IF\n", " (\n", - " A10\n", + " A\n", + " 10\n", " \n", " >\n", " \n", " 0\n", - " ,\n", " ...\n", - " SUM\n", - " (\n", - " A10\n", " ,\n", " \n", - " B10\n", + " B\n", + " 10\n", " )\n", " =\n", - " C10\n", + " C\n", + " 10\n", " +\n", + " 1\n", " \n", " \n", " 10\n", @@ -308,23 +318,23 @@ " =\n", " IF\n", " (\n", - " A11\n", + " A\n", + " 11\n", " \n", " >\n", " \n", " 0\n", - " ,\n", " ...\n", - " SUM\n", - " (\n", - " A11\n", " ,\n", " \n", - " B11\n", + " B\n", + " 11\n", " )\n", " =\n", - " C11\n", + " C\n", + " 11\n", " +\n", + " 1\n", " \n", " \n", " 11\n", @@ -332,23 +342,23 @@ " =\n", " IF\n", " (\n", - " A12\n", + " A\n", + " 12\n", " \n", " >\n", " \n", " 0\n", - " ,\n", " ...\n", - " SUM\n", - " (\n", - " A12\n", " ,\n", " \n", - " B12\n", + " B\n", + " 12\n", " )\n", " =\n", - " C12\n", + " C\n", + " 12\n", " +\n", + " 1\n", " \n", " \n", " 12\n", @@ -356,23 +366,23 @@ " =\n", " IF\n", " (\n", - " A13\n", + " A\n", + " 13\n", " \n", " >\n", " \n", " 0\n", - " ,\n", " ...\n", - " SUM\n", - " (\n", - " A13\n", " ,\n", " \n", - " B13\n", + " B\n", + " 13\n", " )\n", " =\n", - " C13\n", + " C\n", + " 13\n", " +\n", + " 1\n", " \n", " \n", " 13\n", @@ -380,23 +390,23 @@ " =\n", " IF\n", " (\n", - " A14\n", + " A\n", + " 14\n", " \n", " >\n", " \n", " 0\n", - " ,\n", " ...\n", - " SUM\n", - " (\n", - " A14\n", " ,\n", " \n", - " B14\n", + " B\n", + " 14\n", " )\n", " =\n", - " C14\n", + " C\n", + " 14\n", " +\n", + " 1\n", " \n", " \n", " 14\n", @@ -404,68 +414,51 @@ " =\n", " IF\n", " (\n", - " A15\n", + " A\n", + " 15\n", " \n", " >\n", " \n", " 0\n", - " ,\n", " ...\n", - " SUM\n", - " (\n", - " A15\n", " ,\n", " \n", - " B15\n", + " B\n", + " 15\n", " )\n", " =\n", - " C15\n", + " C\n", + " 15\n", " +\n", + " 1\n", " \n", " \n", "\n", - "

15 rows × 27 columns

\n", + "

15 rows × 33 columns

\n", "" ], "text/plain": [ - " 0 1 2 3 4 5 6 7 8 9 ... 17 18 19 20 21 22 23 24 \\\n", - "0 -5 = IF ( A1 > 0 , ... SUM ( A1 , B1 ) = \n", - "1 -4 = IF ( A2 > 0 , ... SUM ( A2 , B2 ) = \n", - "2 -3 = IF ( A3 > 0 , ... SUM ( A3 , B3 ) = \n", - "3 -2 = IF ( A4 > 0 , ... SUM ( A4 , B4 ) = \n", - "4 -1 = IF ( A5 > 0 , ... SUM ( A5 , B5 ) = \n", - "5 0 = IF ( A6 > 0 , ... SUM ( A6 , B6 ) = \n", - "6 1 = IF ( A7 > 0 , ... SUM ( A7 , B7 ) = \n", - "7 2 = IF ( A8 > 0 , ... SUM ( A8 , B8 ) = \n", - "8 3 = IF ( A9 > 0 , ... SUM ( A9 , B9 ) = \n", - "9 4 = IF ( A10 > 0 , ... SUM ( A10 , B10 ) = \n", - "10 5 = IF ( A11 > 0 , ... SUM ( A11 , B11 ) = \n", - "11 6 = IF ( A12 > 0 , ... SUM ( A12 , B12 ) = \n", - "12 7 = IF ( A13 > 0 , ... SUM ( A13 , B13 ) = \n", - "13 8 = IF ( A14 > 0 , ... SUM ( A14 , B14 ) = \n", - "14 9 = IF ( A15 > 0 , ... SUM ( A15 , B15 ) = \n", - "\n", - " 25 26 \n", - "0 C1 + \n", - "1 C2 + \n", - "2 C3 + \n", - "3 C4 + \n", - "4 C5 + \n", - "5 C6 + \n", - "6 C7 + \n", - "7 C8 + \n", - "8 C9 + \n", - "9 C10 + \n", - "10 C11 + \n", - "11 C12 + \n", - "12 C13 + \n", - "13 C14 + \n", - "14 C15 + \n", + " 0 1 2 3 4 5 6 7 8 9 ... 23 24 25 26 27 28 29 30 31 32\n", + "0 -5 = IF ( A 1 > 0 ... , B 1 ) = C 1 + 1\n", + "1 -4 = IF ( A 2 > 0 ... , B 2 ) = C 2 + 1\n", + "2 -3 = IF ( A 3 > 0 ... , B 3 ) = C 3 + 1\n", + "3 -2 = IF ( A 4 > 0 ... , B 4 ) = C 4 + 1\n", + "4 -1 = IF ( A 5 > 0 ... , B 5 ) = C 5 + 1\n", + "5 0 = IF ( A 6 > 0 ... , B 6 ) = C 6 + 1\n", + "6 1 = IF ( A 7 > 0 ... , B 7 ) = C 7 + 1\n", + "7 2 = IF ( A 8 > 0 ... , B 8 ) = C 8 + 1\n", + "8 3 = IF ( A 9 > 0 ... , B 9 ) = C 9 + 1\n", + "9 4 = IF ( A 10 > 0 ... , B 10 ) = C 10 + 1\n", + "10 5 = IF ( A 11 > 0 ... , B 11 ) = C 11 + 1\n", + "11 6 = IF ( A 12 > 0 ... , B 12 ) = C 12 + 1\n", + "12 7 = IF ( A 13 > 0 ... , B 13 ) = C 13 + 1\n", + "13 8 = IF ( A 14 > 0 ... , B 14 ) = C 14 + 1\n", + "14 9 = IF ( A 15 > 0 ... , B 15 ) = C 15 + 1\n", "\n", - "[15 rows x 27 columns]" + "[15 rows x 33 columns]" ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -478,7 +471,6 @@ { "cell_type": "code", "execution_count": null, - "id": "09faa345", "metadata": {}, "outputs": [], "source": [] @@ -486,7 +478,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -500,7 +492,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/add-in/src/api/pids/test/test.json b/add-in/src/api/pids/test/test.json index 2421c98..0e3a391 100644 --- a/add-in/src/api/pids/test/test.json +++ b/add-in/src/api/pids/test/test.json @@ -4,14 +4,16 @@ "=", "IF", "(", - "A1", + "A", + "1", " ", ">", " ", "0", ",", " ", - "A1", + "A", + "1", ",", " ", "0", @@ -19,13 +21,16 @@ "=", "SUM", "(", - "A1", + "A", + "1", ",", " ", - "B1", + "B", + "1", ")", "=", - "C1", + "C", + "1", "+", "1" ], @@ -34,14 +39,16 @@ "=", "IF", "(", - "A2", + "A", + "2", " ", ">", " ", "0", ",", " ", - "A2", + "A", + "2", ",", " ", "0", @@ -49,13 +56,16 @@ "=", "SUM", "(", - "A2", + "A", + "2", ",", " ", - "B2", + "B", + "2", ")", "=", - "C2", + "C", + "2", "+", "1" ], @@ -64,14 +74,16 @@ "=", "IF", "(", - "A3", + "A", + "3", " ", ">", " ", "0", ",", " ", - "A3", + "A", + "3", ",", " ", "0", @@ -79,13 +91,16 @@ "=", "SUM", "(", - "A3", + "A", + "3", ",", " ", - "B3", + "B", + "3", ")", "=", - "C3", + "C", + "3", "+", "1" ], @@ -94,14 +109,16 @@ "=", "IF", "(", - "A4", + "A", + "4", " ", ">", " ", "0", ",", " ", - "A4", + "A", + "4", ",", " ", "0", @@ -109,13 +126,16 @@ "=", "SUM", "(", - "A4", + "A", + "4", ",", " ", - "B4", + "B", + "4", ")", "=", - "C4", + "C", + "4", "+", "1" ], @@ -124,14 +144,16 @@ "=", "IF", "(", - "A5", + "A", + "5", " ", ">", " ", "0", ",", " ", - "A5", + "A", + "5", ",", " ", "0", @@ -139,13 +161,16 @@ "=", "SUM", "(", - "A5", + "A", + "5", ",", " ", - "B5", + "B", + "5", ")", "=", - "C5", + "C", + "5", "+", "1" ], @@ -154,14 +179,16 @@ "=", "IF", "(", - "A6", + "A", + "6", " ", ">", " ", "0", ",", " ", - "A6", + "A", + "6", ",", " ", "0", @@ -169,13 +196,16 @@ "=", "SUM", "(", - "A6", + "A", + "6", ",", " ", - "B6", + "B", + "6", ")", "=", - "C6", + "C", + "6", "+", "1" ], @@ -184,14 +214,16 @@ "=", "IF", "(", - "A7", + "A", + "7", " ", ">", " ", "0", ",", " ", - "A7", + "A", + "7", ",", " ", "0", @@ -199,13 +231,16 @@ "=", "SUM", "(", - "A7", + "A", + "7", ",", " ", - "B7", + "B", + "7", ")", "=", - "C7", + "C", + "7", "+", "1" ], @@ -214,14 +249,16 @@ "=", "IF", "(", - "A8", + "A", + "8", " ", ">", " ", "0", ",", " ", - "A8", + "A", + "8", ",", " ", "0", @@ -229,13 +266,16 @@ "=", "SUM", "(", - "A8", + "A", + "8", ",", " ", - "B8", + "B", + "8", ")", "=", - "C8", + "C", + "8", "+", "1" ], @@ -244,14 +284,16 @@ "=", "IF", "(", - "A9", + "A", + "9", " ", ">", " ", "0", ",", " ", - "A9", + "A", + "9", ",", " ", "0", @@ -259,13 +301,16 @@ "=", "SUM", "(", - "A9", + "A", + "9", ",", " ", - "B9", + "B", + "9", ")", "=", - "C9", + "C", + "9", "+", "1" ], @@ -274,14 +319,16 @@ "=", "IF", "(", - "A10", + "A", + "10", " ", ">", " ", "0", ",", " ", - "A10", + "A", + "10", ",", " ", "0", @@ -289,13 +336,16 @@ "=", "SUM", "(", - "A10", + "A", + "10", ",", " ", - "B10", + "B", + "10", ")", "=", - "C10", + "C", + "10", "+", "1" ], @@ -304,14 +354,16 @@ "=", "IF", "(", - "A11", + "A", + "11", " ", ">", " ", "0", ",", " ", - "A11", + "A", + "11", ",", " ", "0", @@ -319,13 +371,16 @@ "=", "SUM", "(", - "A11", + "A", + "11", ",", " ", - "B11", + "B", + "11", ")", "=", - "C11", + "C", + "11", "+", "1" ], @@ -334,14 +389,16 @@ "=", "IF", "(", - "A12", + "A", + "12", " ", ">", " ", "0", ",", " ", - "A12", + "A", + "12", ",", " ", "0", @@ -349,13 +406,16 @@ "=", "SUM", "(", - "A12", + "A", + "12", ",", " ", - "B12", + "B", + "12", ")", "=", - "C12", + "C", + "12", "+", "1" ], @@ -364,14 +424,16 @@ "=", "IF", "(", - "A13", + "A", + "13", " ", ">", " ", "0", ",", " ", - "A13", + "A", + "13", ",", " ", "0", @@ -379,13 +441,16 @@ "=", "SUM", "(", - "A13", + "A", + "13", ",", " ", - "B13", + "B", + "13", ")", "=", - "C13", + "C", + "13", "+", "1" ], @@ -394,14 +459,16 @@ "=", "IF", "(", - "A14", + "A", + "14", " ", ">", " ", "0", ",", " ", - "A14", + "A", + "14", ",", " ", "0", @@ -409,13 +476,16 @@ "=", "SUM", "(", - "A14", + "A", + "14", ",", " ", - "B14", + "B", + "14", ")", "=", - "C14", + "C", + "14", "+", "1" ], @@ -424,14 +494,16 @@ "=", "IF", "(", - "A15", + "A", + "15", " ", ">", " ", "0", ",", " ", - "A15", + "A", + "15", ",", " ", "0", @@ -439,13 +511,16 @@ "=", "SUM", "(", - "A15", + "A", + "15", ",", " ", - "B15", + "B", + "15", ")", "=", - "C15", + "C", + "15", "+", "1" ] diff --git a/add-in/src/api/pids/test/test_notebook.ipynb b/add-in/src/api/pids/test/test_notebook.ipynb index 099c917..d932587 100644 --- a/add-in/src/api/pids/test/test_notebook.ipynb +++ b/add-in/src/api/pids/test/test_notebook.ipynb @@ -2,10 +2,21 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, - "id": "6b3312fb", + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'lux'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mlux\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'lux'" + ] + } + ], "source": [ "import pandas as pd\n", "import lux" @@ -13,37 +24,443 @@ }, { "cell_type": "code", - "execution_count": 3, - "id": "6792fb2c", + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "b88a86b2b6944557b1271c33658a8c10", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Button(description='Toggle Pandas/Lux', layout=Layout(top='5px', width='140px'), style=ButtonStyle())" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c14981a079c04fb59a7fbd967b82db8b", - "version_major": 2, - "version_minor": 0 - }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
0123456789...23242526272829303132
0-5=IF(A1>0...,B1)=C1+1
1-4=IF(A2>0...,B2)=C2+1
2-3=IF(A3>0...,B3)=C3+1
3-2=IF(A4>0...,B4)=C4+1
4-1=IF(A5>0...,B5)=C5+1
50=IF(A6>0...,B6)=C6+1
61=IF(A7>0...,B7)=C7+1
72=IF(A8>0...,B8)=C8+1
83=IF(A9>0...,B9)=C9+1
94=IF(A10>0...,B10)=C10+1
105=IF(A11>0...,B11)=C11+1
116=IF(A12>0...,B12)=C12+1
127=IF(A13>0...,B13)=C13+1
138=IF(A14>0...,B14)=C14+1
149=IF(A15>0...,B15)=C15+1
\n", + "

15 rows × 33 columns

\n", + "
" + ], "text/plain": [ - "Output()" + " 0 1 2 3 4 5 6 7 8 9 ... 23 24 25 26 27 28 29 30 31 32\n", + "0 -5 = IF ( A 1 > 0 ... , B 1 ) = C 1 + 1\n", + "1 -4 = IF ( A 2 > 0 ... , B 2 ) = C 2 + 1\n", + "2 -3 = IF ( A 3 > 0 ... , B 3 ) = C 3 + 1\n", + "3 -2 = IF ( A 4 > 0 ... , B 4 ) = C 4 + 1\n", + "4 -1 = IF ( A 5 > 0 ... , B 5 ) = C 5 + 1\n", + "5 0 = IF ( A 6 > 0 ... , B 6 ) = C 6 + 1\n", + "6 1 = IF ( A 7 > 0 ... , B 7 ) = C 7 + 1\n", + "7 2 = IF ( A 8 > 0 ... , B 8 ) = C 8 + 1\n", + "8 3 = IF ( A 9 > 0 ... , B 9 ) = C 9 + 1\n", + "9 4 = IF ( A 10 > 0 ... , B 10 ) = C 10 + 1\n", + "10 5 = IF ( A 11 > 0 ... , B 11 ) = C 11 + 1\n", + "11 6 = IF ( A 12 > 0 ... , B 12 ) = C 12 + 1\n", + "12 7 = IF ( A 13 > 0 ... , B 13 ) = C 13 + 1\n", + "13 8 = IF ( A 14 > 0 ... , B 14 ) = C 14 + 1\n", + "14 9 = IF ( A 15 > 0 ... , B 15 ) = C 15 + 1\n", + "\n", + "[15 rows x 33 columns]" ] }, + "execution_count": 6, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" } ], "source": [ @@ -54,7 +471,6 @@ { "cell_type": "code", "execution_count": null, - "id": "3dc21294", "metadata": {}, "outputs": [], "source": [] @@ -62,7 +478,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -76,7 +492,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.7.3" } }, "nbformat": 4, From 52e1b8846ababccda4014ae5ed41a39d4561de84 Mon Sep 17 00:00:00 2001 From: Connor Lien Date: Tue, 6 Sep 2022 00:23:14 -0700 Subject: [PATCH 7/9] Fix starting with 0 bug --- add-in/src/api/pids/pattern/patternMiner.ts | 3 + add-in/src/api/pids/rule/commonSymbolRule.ts | 2 + add-in/src/api/pids/rule/commonWordRule.ts | 24 +- add-in/src/api/pids/test/minerTest.ts | 2 +- add-in/src/api/pids/test/test.json | 655 +++++-------------- add-in/src/api/pids/tokenize/lexer.ts | 51 +- add-in/src/api/pids/utils/stringify.ts | 14 + 7 files changed, 210 insertions(+), 541 deletions(-) diff --git a/add-in/src/api/pids/pattern/patternMiner.ts b/add-in/src/api/pids/pattern/patternMiner.ts index d35f754..8d23610 100644 --- a/add-in/src/api/pids/pattern/patternMiner.ts +++ b/add-in/src/api/pids/pattern/patternMiner.ts @@ -18,8 +18,11 @@ export class PatternMiner { let toRefine: Pattern = translated; let needRefine = true; let refineResult: Pattern = toRefine; + while (needRefine) { + console.log(toRefine.toString()); let refined = this.refine(toRefine); + console.log(refined[1]); if (refined[1] != null) { toRefine = refined[0]; } else { diff --git a/add-in/src/api/pids/rule/commonSymbolRule.ts b/add-in/src/api/pids/rule/commonSymbolRule.ts index 4c8cfd1..0bd0a3b 100644 --- a/add-in/src/api/pids/rule/commonSymbolRule.ts +++ b/add-in/src/api/pids/rule/commonSymbolRule.ts @@ -2,6 +2,7 @@ import { RewriteRule } from "./rewriteRule"; import { PUnion, PSeq, PToken, PEmpty } from "../pattern/pattern"; import { TSymbol } from "../tokenize/token"; import { CommonSeq } from "./commonSeq"; +import { stringifyTensor } from "../utils/stringify"; export class CommonSymbolRule extends RewriteRule { condition(ptn) { @@ -90,6 +91,7 @@ export class CommonSymbolRule extends RewriteRule { idx += 1; } commonSymbolLine.push(line[idx]); + idx += 1; }); commonSymbolsWithPos.push(commonSymbolLine); }); diff --git a/add-in/src/api/pids/rule/commonWordRule.ts b/add-in/src/api/pids/rule/commonWordRule.ts index 51c364e..c4e4825 100644 --- a/add-in/src/api/pids/rule/commonWordRule.ts +++ b/add-in/src/api/pids/rule/commonWordRule.ts @@ -2,6 +2,7 @@ import { RewriteRule } from "./rewriteRule"; import { PUnion, PSeq, PToken, PEmpty } from "../pattern/pattern"; import { TWord } from "../tokenize/token"; import { CommonSeq } from "./commonSeq"; +import { stringifyTensor } from "../utils/stringify"; export class CommonWordRule extends RewriteRule { condition(ptn) { @@ -82,17 +83,20 @@ export class CommonWordRule extends RewriteRule { // Filter out symbols that are not common let commonSymbolsWithPos = []; - symbolsWithPos.forEach((line) => { - let idx = 0; - let commonSymbolLine = []; - commonSymbols.forEach((symbol) => { - while (line[idx] && symbol.token.value !== line[idx][0].token.value) { + if (commonSymbols.length !== 0) { + symbolsWithPos.forEach((line) => { + let idx = 0; + let commonSymbolLine = []; + commonSymbols.forEach((symbol) => { + while (line[idx] && symbol.token.value !== line[idx][0].token.value) { + idx += 1; + } + commonSymbolLine.push(line[idx]); idx += 1; - } - commonSymbolLine.push(line[idx]); + }); + commonSymbolsWithPos.push(commonSymbolLine); }); - commonSymbolsWithPos.push(commonSymbolLine); - }); + } // Package new union split by common symbols let resultSeq = []; @@ -121,6 +125,7 @@ export class CommonWordRule extends RewriteRule { let afterSymbolUnion = []; let nonEmpty = false; commonSymbolsWithPos.forEach((line, lineIdx) => { + console.log(line); let startIdx = line[line.length - 1][1] + 1; const unionLine: any[] = union.content[lineIdx].content; let endIdx = unionLine.length; @@ -133,6 +138,7 @@ export class CommonWordRule extends RewriteRule { afterSymbolUnion.push(new PEmpty()); } }); + if (nonEmpty) { resultSeq.push(new PUnion(afterSymbolUnion)); } diff --git a/add-in/src/api/pids/test/minerTest.ts b/add-in/src/api/pids/test/minerTest.ts index e2b3d91..9d2e7f7 100644 --- a/add-in/src/api/pids/test/minerTest.ts +++ b/add-in/src/api/pids/test/minerTest.ts @@ -3,7 +3,7 @@ import { PatternMiner } from "../pattern/patternMiner"; import { readFile, writeArray } from "../utils/file"; let miner = new PatternMiner(); -let data = testFormMatrix; // readFile("test_ip.csv"); +let data = readFile("test_ip.csv"); let arr = miner.mine(data); console.log(arr); writeArray(arr, "test.json"); diff --git a/add-in/src/api/pids/test/test.json b/add-in/src/api/pids/test/test.json index 0e3a391..8dcaad4 100644 --- a/add-in/src/api/pids/test/test.json +++ b/add-in/src/api/pids/test/test.json @@ -1,527 +1,172 @@ [ [ - "-5", - "=", - "IF", - "(", - "A", - "1", - " ", - ">", - " ", - "0", - ",", - " ", - "A", - "1", - ",", - " ", - "0", - ")", - "=", - "SUM", - "(", - "A", - "1", - ",", - " ", - "B", - "1", - ")", - "=", - "C", - "1", - "+", - "1" + "9755", + ":", + "FE45", + ":", + "E943", + ":", + "6B7F", + ":", + "", + ":", + "8BC5", + ":", + "F0AD", + ":", + "BCBC" ], [ - "-4", - "=", - "IF", - "(", - "A", - "2", - " ", - ">", - " ", - "0", - ",", - " ", - "A", - "2", - ",", - " ", - "0", - ")", - "=", - "SUM", - "(", - "A", - "2", - ",", - " ", - "B", - "2", - ")", - "=", - "C", - "2", - "+", - "1" + "5464", + ":", + "F26A", + ":", + "4EC5", + ":", + "5559", + ":", + "", + ":", + "9710", + ":", + "859F", + ":", + "AD3D" ], [ - "-3", - "=", - "IF", - "(", - "A", - "3", - " ", - ">", - " ", - "0", - ",", - " ", - "A", - "3", - ",", - " ", - "0", - ")", - "=", - "SUM", - "(", - "A", - "3", - ",", - " ", - "B", - "3", - ")", - "=", - "C", - "3", - "+", - "1" + "DF4F", + ":", + "61CB", + ":", + "1A3A", + ":", + "721D", + ":", + "", + ":", + "04AD", + ":", + "CDA9", + ":", + "1188" ], [ - "-2", - "=", - "IF", - "(", - "A", - "4", - " ", - ">", - " ", - "0", - ",", - " ", - "A", - "4", - ",", - " ", - "0", - ")", - "=", - "SUM", - "(", - "A", - "4", - ",", - " ", - "B", - "4", - ")", - "=", - "C", - "4", - "+", - "1" + "7DAE", + ":", + "0162", + ":", + "C620", + ":", + "716C", + ":", + "", + ":", + "6E58", + ":", + "CEB8", + ":", + "194C" ], [ - "-1", - "=", - "IF", - "(", - "A", - "5", - " ", - ">", - " ", - "0", - ",", - " ", - "A", - "5", - ",", - " ", - "0", - ")", - "=", - "SUM", - "(", - "A", - "5", - ",", - " ", - "B", - "5", - ")", - "=", - "C", - "5", - "+", - "1" + "759C", + ":", + "1B54", + ":", + "42CB", + ":", + "C5A2", + ":", + "", + ":", + "6F3C", + ":", + "76B4", + ":", + "A3BD" ], [ - "0", - "=", - "IF", - "(", - "A", - "6", - " ", - ">", - " ", - "0", - ",", - " ", - "A", - "6", - ",", - " ", - "0", - ")", - "=", - "SUM", - "(", - "A", - "6", - ",", - " ", - "B", - "6", - ")", - "=", - "C", - "6", - "+", - "1" + "4222", + ":", + "B2D9", + ":", + "5769", + ":", + "00F3", + ":", + "", + ":", + "F83F", + ":", + "67C0", + ":", + "469B" ], [ - "1", - "=", - "IF", - "(", - "A", - "7", - " ", - ">", - " ", - "0", - ",", - " ", - "A", - "7", - ",", - " ", - "0", - ")", - "=", - "SUM", - "(", - "A", - "7", - ",", - " ", - "B", - "7", - ")", - "=", - "C", - "7", - "+", - "1" + "89BA", + ":", + "F2F0", + ":", + "2CD9", + ":", + "320C", + ":", + "", + ":", + "A69E", + ":", + "16B6", + ":", + "6D00" ], [ - "2", - "=", - "IF", - "(", - "A", - "8", - " ", - ">", - " ", - "0", - ",", - " ", - "A", - "8", - ",", - " ", - "0", - ")", - "=", - "SUM", - "(", - "A", - "8", - ",", - " ", - "B", - "8", - ")", - "=", - "C", - "8", - "+", - "1" + "A69D", + ":", + "046D", + ":", + "8FD4", + ":", + "2E2F", + ":", + "", + ":", + "C305", + ":", + "6619", + ":", + "8734" ], [ - "3", - "=", - "IF", - "(", - "A", - "9", - " ", - ">", - " ", - "0", - ",", - " ", - "A", - "9", - ",", - " ", - "0", - ")", - "=", - "SUM", - "(", - "A", - "9", - ",", - " ", - "B", - "9", - ")", - "=", - "C", - "9", - "+", - "1" + "C8ED", + ":", + "72D8", + ":", + "54F0", + ":", + "F9AD", + ":", + "", + ":", + "A09F", + ":", + "D179", + ":", + "9542" ], [ - "4", - "=", - "IF", - "(", - "A", - "10", - " ", - ">", - " ", - "0", - ",", - " ", - "A", - "10", - ",", - " ", - "0", - ")", - "=", - "SUM", - "(", - "A", - "10", - ",", - " ", - "B", - "10", - ")", - "=", - "C", - "10", - "+", - "1" - ], - [ - "5", - "=", - "IF", - "(", - "A", - "11", - " ", - ">", - " ", - "0", - ",", - " ", - "A", - "11", - ",", - " ", - "0", - ")", - "=", - "SUM", - "(", - "A", - "11", - ",", - " ", - "B", - "11", - ")", - "=", - "C", - "11", - "+", - "1" - ], - [ - "6", - "=", - "IF", - "(", - "A", - "12", - " ", - ">", - " ", - "0", - ",", - " ", - "A", - "12", - ",", - " ", - "0", - ")", - "=", - "SUM", - "(", - "A", - "12", - ",", - " ", - "B", - "12", - ")", - "=", - "C", - "12", - "+", - "1" - ], - [ - "7", - "=", - "IF", - "(", - "A", - "13", - " ", - ">", - " ", - "0", - ",", - " ", - "A", - "13", - ",", - " ", - "0", - ")", - "=", - "SUM", - "(", - "A", - "13", - ",", - " ", - "B", - "13", - ")", - "=", - "C", - "13", - "+", - "1" - ], - [ - "8", - "=", - "IF", - "(", - "A", - "14", - " ", - ">", - " ", - "0", - ",", - " ", - "A", - "14", - ",", - " ", - "0", - ")", - "=", - "SUM", - "(", - "A", - "14", - ",", - " ", - "B", - "14", - ")", - "=", - "C", - "14", - "+", - "1" - ], - [ - "9", - "=", - "IF", - "(", - "A", - "15", - " ", - ">", - " ", - "0", - ",", - " ", - "A", - "15", - ",", - " ", - "0", - ")", - "=", - "SUM", - "(", - "A", - "15", - ",", - " ", - "B", - "15", - ")", - "=", - "C", - "15", - "+", - "1" + "A3A3", + ":", + "DCC2", + ":", + "C369", + ":", + "886A", + ":", + "", + ":", + "F5F8", + ":", + "652F", + ":", + "09B5" ] ] \ No newline at end of file diff --git a/add-in/src/api/pids/tokenize/lexer.ts b/add-in/src/api/pids/tokenize/lexer.ts index d085543..780077a 100644 --- a/add-in/src/api/pids/tokenize/lexer.ts +++ b/add-in/src/api/pids/tokenize/lexer.ts @@ -13,17 +13,6 @@ const conds = [ } }, }, - // { - // name: "double", - // tester: function (tested) { - // var regResult = tested.match(/[+-]?([0-9]*[.])?[0-9]+/); - // if (regResult && regResult[0] === tested) { - // return lexerResults.possible; - // } else { - // return lexerResults.none; - // } - // }, - // }, { name: "int", tester: function (tested) { @@ -61,22 +50,32 @@ const conds = [ export class Lexer { static lex(str) { let tokens = lexer(str, conds); - return tokens.map((token) => { - switch (token.type) { - case "word": - return new TWord(token.value); - case "int": - return new TInt(parseInt(token.value)); - case "double": - return new TDouble(parseFloat(token.value)); - case "symbol": - return new TSymbol(token.value); - case "space": - return new TSpace(); - default: - console.log("Token type does not exist!"); - return new TSymbol(""); + let result = []; + tokens.forEach((token) => { + let type = token.type; + let value = token.value; + if (type === "word") { + result.push(new TWord(value)); + } else if (type === "int") { + // To handle the edge case of a number starting with a 0, we don't want to remove the 0 + while (value[0] === "0") { + result.push(new TInt(0)); + value = value.slice(1); + } + if (value.length !== 0) { + result.push(new TInt(parseInt(value))); + } + } else if (type === "double") { + result.push(new TDouble(parseFloat(value))); + } else if (type === "symbol") { + result.push(new TSymbol(value)); + } else if (type === "space") { + result.push(new TSpace()); + } else { + console.log("Token type does not exist!"); + result.push(new TSymbol("")); } }); + return result; } } diff --git a/add-in/src/api/pids/utils/stringify.ts b/add-in/src/api/pids/utils/stringify.ts index 619b03c..b7a3f3c 100644 --- a/add-in/src/api/pids/utils/stringify.ts +++ b/add-in/src/api/pids/utils/stringify.ts @@ -13,3 +13,17 @@ export function stringifyMatrix(mat) { } return result; } + +export function stringifyTensor(tensor, deg) { + if (deg === 1) { + return stringifyArray(tensor); + } + if (deg === 2) { + return stringifyMatrix(tensor); + } + let result = []; + for (let i = 0; i < tensor.length; i += 1) { + result.push(stringifyTensor(tensor[i], deg - 1)); + } + return result; +} From 28437f27ebfc52e79ee18a60bd7dc18917cc0467 Mon Sep 17 00:00:00 2001 From: Connor Lien Date: Thu, 8 Sep 2022 00:26:17 -0700 Subject: [PATCH 8/9] Combine constant columns --- add-in/src/api/pids/pattern/extractPattern.ts | 26 +- add-in/src/api/pids/pattern/patternMiner.ts | 46 +- add-in/src/api/pids/pattern/postProcessing.ts | 82 +++ add-in/src/api/pids/rule/adacentTokenRule.ts | 30 ++ add-in/src/api/pids/rule/commonSymbolRule.ts | 5 + add-in/src/api/pids/rule/commonWordRule.ts | 28 +- add-in/src/api/pids/rule/sameItemRule.ts | 1 + add-in/src/api/pids/test/extractionTest.ts | 6 +- add-in/src/api/pids/test/minerTest.ts | 2 +- add-in/src/api/pids/test/test.json | 370 ++++++++------ add-in/src/api/pids/test/testMatrix.ts | 31 ++ add-in/src/api/pids/test/test_notebook.ipynb | 472 ++---------------- add-in/src/api/pids/test/tokenizeTest.ts | 2 +- add-in/src/api/pids/utils/matrix.ts | 6 + 14 files changed, 435 insertions(+), 672 deletions(-) create mode 100644 add-in/src/api/pids/pattern/postProcessing.ts create mode 100644 add-in/src/api/pids/rule/adacentTokenRule.ts create mode 100644 add-in/src/api/pids/utils/matrix.ts diff --git a/add-in/src/api/pids/pattern/extractPattern.ts b/add-in/src/api/pids/pattern/extractPattern.ts index d374f5c..f0d3668 100644 --- a/add-in/src/api/pids/pattern/extractPattern.ts +++ b/add-in/src/api/pids/pattern/extractPattern.ts @@ -1,4 +1,5 @@ import { PatternMiner } from "./patternMiner"; +import { transposeMatrix } from "../utils/matrix"; /* Extract a pattern from a formula matrix that is stored in row-major. @@ -7,26 +8,13 @@ import { PatternMiner } from "./patternMiner"; export function extractPattern(formMatrix) { let miner = new PatternMiner(); miner.sampleSize = 2000; - let transpose = transposeArray(formMatrix); + let transpose = transposeMatrix(formMatrix); let patterns = []; for (let colIdx = 0; colIdx < transpose.length; colIdx += 1) { - let pattern = miner.mine(transpose[colIdx]); - patterns.push(pattern); - console.log(pattern); + let col = transposeMatrix([transpose[colIdx]]); + let pattern = miner.mine(col); + let transposed = transposeMatrix(pattern); + patterns.push(...transposed); } - return patterns; -} - -/* - Transpose the array to change it from row to column major. - We evaluate PIDS along each column. -*/ -function transposeArray(formMatrix) { - let result = JSON.parse(JSON.stringify(formMatrix)); - for (let i = 0; i < formMatrix.length; i += 1) { - for (let j = 0; j < formMatrix[i].length; j += 1) { - result[j][i] = formMatrix[i][j]; - } - } - return result; + return transposeMatrix(patterns); } diff --git a/add-in/src/api/pids/pattern/patternMiner.ts b/add-in/src/api/pids/pattern/patternMiner.ts index 8d23610..7fca46b 100644 --- a/add-in/src/api/pids/pattern/patternMiner.ts +++ b/add-in/src/api/pids/pattern/patternMiner.ts @@ -3,14 +3,13 @@ import { Pattern, PUnion, PSeq, PToken } from "./pattern"; import { CommonSymbolRule } from "../rule/commonSymbolRule"; import { CommonWordRule } from "../rule/commonWordRule"; import { SameItemRule } from "../rule/sameItemRule"; +import { repack, combineConstantCols } from "./postProcessing"; export class PatternMiner { sampleSize = 500; rules = [new CommonSymbolRule(), new CommonWordRule(), new SameItemRule()]; - getPSeqString = (seq) => seq.content.map((token) => token.toString()).join(""); - mine(lines) { let tokens = lines.map((line) => Tokenizer.tokenize(line)); let translated = new PUnion(tokens.map((line) => new PSeq(line.map((elem) => new PToken(elem))))); @@ -20,9 +19,7 @@ export class PatternMiner { let refineResult: Pattern = toRefine; while (needRefine) { - console.log(toRefine.toString()); let refined = this.refine(toRefine); - console.log(refined[1]); if (refined[1] != null) { toRefine = refined[0]; } else { @@ -30,7 +27,8 @@ export class PatternMiner { refineResult = refined[0]; } } - return this.repack(refineResult, lines.length); + let repacked = repack(refineResult, lines.length); + return combineConstantCols(repacked); } refine(root) { @@ -45,42 +43,4 @@ export class PatternMiner { } return [current, null]; } - - repack(pattern, entries) { - if (!(pattern instanceof PSeq)) { - pattern = new PSeq(pattern); - } - let result = []; - for (let idx = 0; idx < entries; idx += 1) { - result.push(this.getPatternArray(pattern, idx)); - } - return result; - } - - getPatternArray(pattern, idx) { - if (pattern instanceof PToken) { - return [pattern.token.toString()]; - } - let hasSubpattern = false; - pattern.content.forEach((subpattern) => { - if (subpattern instanceof PSeq || subpattern instanceof PUnion) { - hasSubpattern = true; - } - }); - if (hasSubpattern && pattern instanceof PSeq) { - let line = []; - pattern.content.forEach((subpattern) => { - let arr = this.getPatternArray(subpattern, idx); - line.push(...arr); - }); - return line; - } else if (hasSubpattern && pattern instanceof PUnion) { - return this.getPatternArray(pattern.content[idx], idx); - } else if (pattern instanceof PSeq) { - return [this.getPSeqString(pattern)]; - } else if (pattern instanceof PUnion) { - return [pattern.content[idx]]; - } - return []; - } } diff --git a/add-in/src/api/pids/pattern/postProcessing.ts b/add-in/src/api/pids/pattern/postProcessing.ts new file mode 100644 index 0000000..e620d9f --- /dev/null +++ b/add-in/src/api/pids/pattern/postProcessing.ts @@ -0,0 +1,82 @@ +import { PUnion, PSeq, PToken } from "./pattern"; +import { transposeMatrix } from "../utils/matrix"; + +const getPSeqString = (seq) => seq.content.map((token) => token.toString()).join(""); + +export function repack(pattern, entries) { + if (!(pattern instanceof PSeq || pattern instanceof PUnion)) { + pattern = new PSeq(pattern); + } + let result = []; + for (let idx = 0; idx < entries; idx += 1) { + result.push(getPatternArray(pattern, idx)); + } + return result; +} + +function getPatternArray(pattern, idx) { + if (pattern instanceof PToken) { + return [pattern.token.toString()]; + } + let hasSubpattern = false; + pattern.content.forEach((subpattern) => { + if (subpattern instanceof PSeq || subpattern instanceof PUnion) { + hasSubpattern = true; + } + }); + if (hasSubpattern && pattern instanceof PSeq) { + let line = []; + pattern.content.forEach((subpattern) => { + let arr = getPatternArray(subpattern, idx); + line.push(...arr); + }); + return line; + } else if (hasSubpattern && pattern instanceof PUnion) { + return getPatternArray(pattern.content[idx], idx); + } else if (pattern instanceof PSeq) { + return [getPSeqString(pattern)]; + } else if (pattern instanceof PUnion) { + return [pattern.content[idx]]; + } + return []; +} + +export function combineConstantCols(mat: string[][]) { + /* Transpose matrix for better cache locality. */ + let constantCols = new Set(); + for (let x = 0; x < mat[0].length; x += 1) { + let constant = true; + let firstVal = mat[0][x]; + for (let y = 0; y < mat.length; y += 1) { + if (mat[y][x] !== firstVal) { + constant = false; + break; + } + } + if (constant) { + constantCols.add(x); + } + } + + let result = []; + for (let y = 0; y < mat.length; y += 1) { + let resultRow = []; + let runningString = ""; + for (let x = 0; x < mat[0].length; x += 1) { + if (constantCols.has(x)) { + runningString += mat[y][x]; + } else { + if (runningString.length !== 0) { + resultRow.push(runningString); + runningString = ""; + } + resultRow.push(mat[y][x]); + } + } + if (runningString.length !== 0) { + resultRow.push(runningString); + } + result.push(resultRow); + } + return result; +} diff --git a/add-in/src/api/pids/rule/adacentTokenRule.ts b/add-in/src/api/pids/rule/adacentTokenRule.ts new file mode 100644 index 0000000..b20eedb --- /dev/null +++ b/add-in/src/api/pids/rule/adacentTokenRule.ts @@ -0,0 +1,30 @@ +import { RewriteRule } from "./rewriteRule"; +import { PToken, Pattern } from "../pattern/pattern"; +import { TWord } from "../tokenize/token"; + +/* If two tokens are adjacent to each other, combine them. */ +export class AdjacentTokenRule extends RewriteRule { + condition(_) { + return true; + } + + update(ptn: Pattern) { + if (ptn instanceof Array) { + let result = []; + let runningToken = ""; + ptn.forEach((subpattern) => { + if (subpattern instanceof PToken) { + runningToken += subpattern.token.toString(); + } else { + if (runningToken.length !== 0) { + result.push(new PToken(new TWord(runningToken))); + runningToken = ""; + } + result.push(subpattern); + } + }); + return result; + } + return ptn; + } +} diff --git a/add-in/src/api/pids/rule/commonSymbolRule.ts b/add-in/src/api/pids/rule/commonSymbolRule.ts index 0bd0a3b..10f3e93 100644 --- a/add-in/src/api/pids/rule/commonSymbolRule.ts +++ b/add-in/src/api/pids/rule/commonSymbolRule.ts @@ -81,6 +81,11 @@ export class CommonSymbolRule extends RewriteRule { const eq = (a: Array, b: Array) => (a[0] as PToken).toString() === (b[0] as PToken).toString(); let commonSymbols = commonSeq.find(validLines, eq).map((i) => i[0]); + // If there are no common symbols, do nothing + if (commonSymbols.length === 0) { + return union; + } + // Filter out symbols that are not common let commonSymbolsWithPos = []; symbolsWithPos.forEach((line) => { diff --git a/add-in/src/api/pids/rule/commonWordRule.ts b/add-in/src/api/pids/rule/commonWordRule.ts index c4e4825..7ba4b7b 100644 --- a/add-in/src/api/pids/rule/commonWordRule.ts +++ b/add-in/src/api/pids/rule/commonWordRule.ts @@ -81,22 +81,25 @@ export class CommonWordRule extends RewriteRule { const eq = (a: Array, b: Array) => (a[0] as PToken).toString() === (b[0] as PToken).toString(); let commonSymbols = commonSeq.find(validLines, eq).map((i) => i[0]); + // If there are no common symbols, do nothing + if (commonSymbols.length === 0) { + return union; + } + // Filter out symbols that are not common let commonSymbolsWithPos = []; - if (commonSymbols.length !== 0) { - symbolsWithPos.forEach((line) => { - let idx = 0; - let commonSymbolLine = []; - commonSymbols.forEach((symbol) => { - while (line[idx] && symbol.token.value !== line[idx][0].token.value) { - idx += 1; - } - commonSymbolLine.push(line[idx]); + symbolsWithPos.forEach((line) => { + let idx = 0; + let commonSymbolLine = []; + commonSymbols.forEach((symbol) => { + while (line[idx] && symbol.token.value !== line[idx][0].token.value) { idx += 1; - }); - commonSymbolsWithPos.push(commonSymbolLine); + } + commonSymbolLine.push(line[idx]); + idx += 1; }); - } + commonSymbolsWithPos.push(commonSymbolLine); + }); // Package new union split by common symbols let resultSeq = []; @@ -125,7 +128,6 @@ export class CommonWordRule extends RewriteRule { let afterSymbolUnion = []; let nonEmpty = false; commonSymbolsWithPos.forEach((line, lineIdx) => { - console.log(line); let startIdx = line[line.length - 1][1] + 1; const unionLine: any[] = union.content[lineIdx].content; let endIdx = unionLine.length; diff --git a/add-in/src/api/pids/rule/sameItemRule.ts b/add-in/src/api/pids/rule/sameItemRule.ts index 780a93e..6428b4e 100644 --- a/add-in/src/api/pids/rule/sameItemRule.ts +++ b/add-in/src/api/pids/rule/sameItemRule.ts @@ -1,6 +1,7 @@ import { RewriteRule } from "./rewriteRule"; import { PUnion, PSeq, PToken, PEmpty, Pattern } from "../pattern/pattern"; +/* If a union has all identical items, combine them into a single Token. */ export class SameItemRule extends RewriteRule { condition(ptn) { if (ptn instanceof PUnion || ptn instanceof PSeq) { diff --git a/add-in/src/api/pids/test/extractionTest.ts b/add-in/src/api/pids/test/extractionTest.ts index 3d1d64c..a774f6c 100644 --- a/add-in/src/api/pids/test/extractionTest.ts +++ b/add-in/src/api/pids/test/extractionTest.ts @@ -1,4 +1,8 @@ import { testFormMatrix } from "./testMatrix"; import { extractPattern } from "../pattern/extractPattern"; +import { readFile, writeArray } from "../utils/file"; -extractPattern(testFormMatrix); +let data = testFormMatrix; //readFile("test_ip.csv"); +let arr = extractPattern(data); +console.log(arr); +writeArray(arr, "test.json"); diff --git a/add-in/src/api/pids/test/minerTest.ts b/add-in/src/api/pids/test/minerTest.ts index 9d2e7f7..bf27e8b 100644 --- a/add-in/src/api/pids/test/minerTest.ts +++ b/add-in/src/api/pids/test/minerTest.ts @@ -3,7 +3,7 @@ import { PatternMiner } from "../pattern/patternMiner"; import { readFile, writeArray } from "../utils/file"; let miner = new PatternMiner(); -let data = readFile("test_ip.csv"); +let data = testFormMatrix; //readFile("test_ip.csv"); let arr = miner.mine(data); console.log(arr); writeArray(arr, "test.json"); diff --git a/add-in/src/api/pids/test/test.json b/add-in/src/api/pids/test/test.json index 8dcaad4..31bce29 100644 --- a/add-in/src/api/pids/test/test.json +++ b/add-in/src/api/pids/test/test.json @@ -1,172 +1,242 @@ [ [ - "9755", - ":", - "FE45", - ":", - "E943", - ":", - "6B7F", - ":", - "", - ":", - "8BC5", - ":", - "F0AD", - ":", - "BCBC" + "-5", + "=IF(A", + "1", + " > 0, A", + "1", + ", 0)", + "=SUM(A", + "1", + ", B", + "1", + ")", + "=C", + "1", + "+1" ], [ - "5464", - ":", - "F26A", - ":", - "4EC5", - ":", - "5559", - ":", - "", - ":", - "9710", - ":", - "859F", - ":", - "AD3D" + "-4", + "=IF(A", + "2", + " > 0, A", + "2", + ", 0)", + "=SUM(A", + "2", + ", B", + "2", + ")", + "=C", + "2", + "+1" ], [ - "DF4F", - ":", - "61CB", - ":", - "1A3A", - ":", - "721D", - ":", - "", - ":", - "04AD", - ":", - "CDA9", - ":", - "1188" + "-3", + "=IF(A", + "3", + " > 0, A", + "3", + ", 0)", + "=SUM(A", + "3", + ", B", + "3", + ")", + "=C", + "3", + "+1" ], [ - "7DAE", - ":", - "0162", - ":", - "C620", - ":", - "716C", - ":", - "", - ":", - "6E58", - ":", - "CEB8", - ":", - "194C" + "-2", + "=IF(A", + "4", + " > 0, A", + "4", + ", 0)", + "=SUM(A", + "4", + ", B", + "4", + ")", + "=C", + "4", + "+1" ], [ - "759C", - ":", - "1B54", - ":", - "42CB", - ":", - "C5A2", - ":", - "", - ":", - "6F3C", - ":", - "76B4", - ":", - "A3BD" + "-1", + "=IF(A", + "5", + " > 0, A", + "5", + ", 0)", + "=SUM(A", + "5", + ", B", + "5", + ")", + "=C", + "5", + "+1" ], [ - "4222", - ":", - "B2D9", - ":", - "5769", - ":", - "00F3", - ":", - "", - ":", - "F83F", - ":", - "67C0", - ":", - "469B" + "0", + "=IF(A", + "6", + " > 0, A", + "6", + ", 0)", + "=SUM(A", + "6", + ", B", + "6", + ")", + "=C", + "6", + "+1" ], [ - "89BA", - ":", - "F2F0", - ":", - "2CD9", - ":", - "320C", - ":", - "", - ":", - "A69E", - ":", - "16B6", - ":", - "6D00" + "1", + "=IF(A", + "7", + " > 0, A", + "7", + ", 0)", + "=SUM(A", + "7", + ", B", + "7", + ")", + "=C", + "7", + "+1" ], [ - "A69D", - ":", - "046D", - ":", - "8FD4", - ":", - "2E2F", - ":", - "", - ":", - "C305", - ":", - "6619", - ":", - "8734" + "2", + "=IF(A", + "8", + " > 0, A", + "8", + ", 0)", + "=SUM(A", + "8", + ", B", + "8", + ")", + "=C", + "8", + "+1" ], [ - "C8ED", - ":", - "72D8", - ":", - "54F0", - ":", - "F9AD", - ":", - "", - ":", - "A09F", - ":", - "D179", - ":", - "9542" + "3", + "=IF(A", + "9", + " > 0, A", + "9", + ", 0)", + "=SUM(A", + "9", + ", B", + "9", + ")", + "=C", + "9", + "+1" ], [ - "A3A3", - ":", - "DCC2", - ":", - "C369", - ":", - "886A", - ":", - "", - ":", - "F5F8", - ":", - "652F", - ":", - "09B5" + "4", + "=IF(A", + "10", + " > 0, A", + "10", + ", 0)", + "=SUM(A", + "10", + ", B", + "10", + ")", + "=C", + "10", + "+1" + ], + [ + "5", + "=IF(A", + "11", + " > 0, A", + "11", + ", 0)", + "=SUM(A", + "11", + ", B", + "11", + ")", + "=C", + "11", + "+1" + ], + [ + "6", + "=IF(A", + "12", + " > 0, A", + "12", + ", 0)", + "=SUM(A", + "12", + ", B", + "12", + ")", + "=C", + "12", + "+1" + ], + [ + "7", + "=IF(A", + "13", + " > 0, A", + "13", + ", 0)", + "=SUM(A", + "13", + ", B", + "13", + ")", + "=C", + "13", + "+1" + ], + [ + "8", + "=IF(A", + "14", + " > 0, A", + "14", + ", 0)", + "=SUM(A", + "14", + ", B", + "14", + ")", + "=C", + "14", + "+1" + ], + [ + "9", + "=IF(A", + "15", + " > 0, A", + "15", + ", 0)", + "=SUM(A", + "15", + ", B", + "15", + ")", + "=C", + "15", + "+1" ] ] \ No newline at end of file diff --git a/add-in/src/api/pids/test/testMatrix.ts b/add-in/src/api/pids/test/testMatrix.ts index 76eff77..5bcafc3 100644 --- a/add-in/src/api/pids/test/testMatrix.ts +++ b/add-in/src/api/pids/test/testMatrix.ts @@ -15,3 +15,34 @@ export const testFormMatrix = [ ["8", "=IF(A14 > 0, A14, 0)", "=SUM(A14, B14)", "=C14+1"], ["9", "=IF(A15 > 0, A15, 0)", "=SUM(A15, B15)", "=C15+1"], ]; + +export const testFormMatrix2 = [ + ["=SUM(A1, B1)"], + ["=SUM(A3, B1)"], + ["=SUM(A5, B1)"], + ["=SUM(A7, B1)"], + ["=SUM(A9, B1)"], + ["=SUM(A11, B1)"], + ["=SUM(A13, B1)"], + ["=SUM(A15, B1)"], + ["=SUM(A17, B1)"], + ["=SUM(A19, B1)"], +]; + +export const testFormMatrix3 = [ + ["-51", "=IF(A1 > 0, A1, 0)", "=SUM(A1, B1)", "=C1+1"], + ["-490", "=IF(A2 > 0, A2, 0)", "=SUM(A2, B2)", "=C2+1"], + ["-3", "=IF(A3 > 0, A3, 0)", "=SUM(A3, B3)", "=C3+1"], + ["-2302", "=IF(A4 > 0, A4, 0)", "=SUM(A4, B4)", "=C4+1"], + ["-1", "=IF(A5 > 0, A5, 0)", "=SUM(A5, B5)", "=C5+1"], + ["0", "=IF(A6 > 0, A6, 0)", "=SUM(A6, B6)", "=C6+1"], + ["156", "=IF(A7 > 0, A7, 0)", "=SUM(A7, B7)", "=C7+1"], + ["2535", "=IF(A8 > 0, A8, 0)", "=SUM(A8, B8)", "=C8+1"], + ["3123454", "=IF(A9 > 0, A9, 0)", "=SUM(A9, B9)", "=C9+1"], + ["4234", "=IF(A10 > 0, A10, 0)", "=SUM(A10, B10)", "=C10+1"], + ["52", "=IF(A11 > 0, A11, 0)", "=SUM(A11, B11)", "=C11+1"], + ["67777777777", "=IF(A12 > 0, A12, 0)", "=SUM(A12, B12)", "=C12+1"], + ["347", "=IF(A13 > 0, A13, 0)", "=SUM(A13, B13)", "=C13+1"], + ["875", "=IF(A14 > 0, A14, 0)", "=SUM(A14, B14)", "=C14+1"], + ["95", "=IF(A15 > 0, A15, 0)", "=SUM(A15, B15)", "=C15+1"], +]; diff --git a/add-in/src/api/pids/test/test_notebook.ipynb b/add-in/src/api/pids/test/test_notebook.ipynb index d932587..91f7c8d 100644 --- a/add-in/src/api/pids/test/test_notebook.ipynb +++ b/add-in/src/api/pids/test/test_notebook.ipynb @@ -2,21 +2,10 @@ "cells": [ { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, + "id": "ae79869c", "metadata": {}, - "outputs": [ - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'lux'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mlux\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'lux'" - ] - } - ], + "outputs": [], "source": [ "import pandas as pd\n", "import lux" @@ -25,442 +14,36 @@ { "cell_type": "code", "execution_count": 6, + "id": "9ad77165", "metadata": {}, "outputs": [ { "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
0123456789...23242526272829303132
0-5=IF(A1>0...,B1)=C1+1
1-4=IF(A2>0...,B2)=C2+1
2-3=IF(A3>0...,B3)=C3+1
3-2=IF(A4>0...,B4)=C4+1
4-1=IF(A5>0...,B5)=C5+1
50=IF(A6>0...,B6)=C6+1
61=IF(A7>0...,B7)=C7+1
72=IF(A8>0...,B8)=C8+1
83=IF(A9>0...,B9)=C9+1
94=IF(A10>0...,B10)=C10+1
105=IF(A11>0...,B11)=C11+1
116=IF(A12>0...,B12)=C12+1
127=IF(A13>0...,B13)=C13+1
138=IF(A14>0...,B14)=C14+1
149=IF(A15>0...,B15)=C15+1
\n", - "

15 rows × 33 columns

\n", - "
" - ], + "application/vnd.jupyter.widget-view+json": { + "model_id": "e8b6a9f824cf43f9b69cd67232974532", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Button(description='Toggle Pandas/Lux', layout=Layout(top='5px', width='140px'), style=ButtonStyle())" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "954c8f3cb46243fbae07819ce4130698", + "version_major": 2, + "version_minor": 0 + }, "text/plain": [ - " 0 1 2 3 4 5 6 7 8 9 ... 23 24 25 26 27 28 29 30 31 32\n", - "0 -5 = IF ( A 1 > 0 ... , B 1 ) = C 1 + 1\n", - "1 -4 = IF ( A 2 > 0 ... , B 2 ) = C 2 + 1\n", - "2 -3 = IF ( A 3 > 0 ... , B 3 ) = C 3 + 1\n", - "3 -2 = IF ( A 4 > 0 ... , B 4 ) = C 4 + 1\n", - "4 -1 = IF ( A 5 > 0 ... , B 5 ) = C 5 + 1\n", - "5 0 = IF ( A 6 > 0 ... , B 6 ) = C 6 + 1\n", - "6 1 = IF ( A 7 > 0 ... , B 7 ) = C 7 + 1\n", - "7 2 = IF ( A 8 > 0 ... , B 8 ) = C 8 + 1\n", - "8 3 = IF ( A 9 > 0 ... , B 9 ) = C 9 + 1\n", - "9 4 = IF ( A 10 > 0 ... , B 10 ) = C 10 + 1\n", - "10 5 = IF ( A 11 > 0 ... , B 11 ) = C 11 + 1\n", - "11 6 = IF ( A 12 > 0 ... , B 12 ) = C 12 + 1\n", - "12 7 = IF ( A 13 > 0 ... , B 13 ) = C 13 + 1\n", - "13 8 = IF ( A 14 > 0 ... , B 14 ) = C 14 + 1\n", - "14 9 = IF ( A 15 > 0 ... , B 15 ) = C 15 + 1\n", - "\n", - "[15 rows x 33 columns]" + "Output()" ] }, - "execution_count": 6, "metadata": {}, - "output_type": "execute_result" + "output_type": "display_data" } ], "source": [ @@ -471,6 +54,7 @@ { "cell_type": "code", "execution_count": null, + "id": "f41c4271", "metadata": {}, "outputs": [], "source": [] @@ -478,7 +62,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -492,7 +76,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.9.7" } }, "nbformat": 4, diff --git a/add-in/src/api/pids/test/tokenizeTest.ts b/add-in/src/api/pids/test/tokenizeTest.ts index c641a10..eee3294 100644 --- a/add-in/src/api/pids/test/tokenizeTest.ts +++ b/add-in/src/api/pids/test/tokenizeTest.ts @@ -1,4 +1,4 @@ -import { testFormMatrix1 } from "./testMatrix"; +import { testFormMatrix } from "./testMatrix"; import { stringifyArray } from "../utils/stringify"; import { Tokenizer } from "../tokenize/tokenizer"; diff --git a/add-in/src/api/pids/utils/matrix.ts b/add-in/src/api/pids/utils/matrix.ts new file mode 100644 index 0000000..54660c6 --- /dev/null +++ b/add-in/src/api/pids/utils/matrix.ts @@ -0,0 +1,6 @@ +/* + Transpose a matrix array. +*/ +export function transposeMatrix(formMatrix) { + return formMatrix[0].map((_, colIndex) => formMatrix.map((row) => row[colIndex])); +} From 7edb57392a482aa08b4d19f31df859e056e1b121 Mon Sep 17 00:00:00 2001 From: Connor Lien Date: Thu, 8 Sep 2022 16:39:45 -0700 Subject: [PATCH 9/9] Additional test data --- add-in/src/api/pids/test/test.json | 240 ++------- add-in/src/api/pids/test/test2.json | 242 +++++++++ add-in/src/api/pids/test/test3.json | 527 +++++++++++++++++++ add-in/src/api/pids/test/test4.json | 452 ++++++++++++++++ add-in/src/api/pids/test/test5.json | 172 ++++++ add-in/src/api/pids/test/test_notebook.ipynb | 170 +++++- 6 files changed, 1603 insertions(+), 200 deletions(-) create mode 100644 add-in/src/api/pids/test/test2.json create mode 100644 add-in/src/api/pids/test/test3.json create mode 100644 add-in/src/api/pids/test/test4.json create mode 100644 add-in/src/api/pids/test/test5.json diff --git a/add-in/src/api/pids/test/test.json b/add-in/src/api/pids/test/test.json index 31bce29..1965186 100644 --- a/add-in/src/api/pids/test/test.json +++ b/add-in/src/api/pids/test/test.json @@ -1,242 +1,92 @@ [ [ "-5", - "=IF(A", - "1", - " > 0, A", - "1", - ", 0)", - "=SUM(A", - "1", - ", B", - "1", - ")", - "=C", - "1", - "+1" + "=IF(A1 > 0, A1, 0)", + "=SUM(A1, B1)", + "=C1+1" ], [ "-4", - "=IF(A", - "2", - " > 0, A", - "2", - ", 0)", - "=SUM(A", - "2", - ", B", - "2", - ")", - "=C", - "2", - "+1" + "=IF(A2 > 0, A2, 0)", + "=SUM(A2, B2)", + "=C2+1" ], [ "-3", - "=IF(A", - "3", - " > 0, A", - "3", - ", 0)", - "=SUM(A", - "3", - ", B", - "3", - ")", - "=C", - "3", - "+1" + "=IF(A3 > 0, A3, 0)", + "=SUM(A3, B3)", + "=C3+1" ], [ "-2", - "=IF(A", - "4", - " > 0, A", - "4", - ", 0)", - "=SUM(A", - "4", - ", B", - "4", - ")", - "=C", - "4", - "+1" + "=IF(A4 > 0, A4, 0)", + "=SUM(A4, B4)", + "=C4+1" ], [ "-1", - "=IF(A", - "5", - " > 0, A", - "5", - ", 0)", - "=SUM(A", - "5", - ", B", - "5", - ")", - "=C", - "5", - "+1" + "=IF(A5 > 0, A5, 0)", + "=SUM(A5, B5)", + "=C5+1" ], [ "0", - "=IF(A", - "6", - " > 0, A", - "6", - ", 0)", - "=SUM(A", - "6", - ", B", - "6", - ")", - "=C", - "6", - "+1" + "=IF(A6 > 0, A6, 0)", + "=SUM(A6, B6)", + "=C6+1" ], [ "1", - "=IF(A", - "7", - " > 0, A", - "7", - ", 0)", - "=SUM(A", - "7", - ", B", - "7", - ")", - "=C", - "7", - "+1" + "=IF(A7 > 0, A7, 0)", + "=SUM(A7, B7)", + "=C7+1" ], [ "2", - "=IF(A", - "8", - " > 0, A", - "8", - ", 0)", - "=SUM(A", - "8", - ", B", - "8", - ")", - "=C", - "8", - "+1" + "=IF(A8 > 0, A8, 0)", + "=SUM(A8, B8)", + "=C8+1" ], [ "3", - "=IF(A", - "9", - " > 0, A", - "9", - ", 0)", - "=SUM(A", - "9", - ", B", - "9", - ")", - "=C", - "9", - "+1" + "=IF(A9 > 0, A9, 0)", + "=SUM(A9, B9)", + "=C9+1" ], [ "4", - "=IF(A", - "10", - " > 0, A", - "10", - ", 0)", - "=SUM(A", - "10", - ", B", - "10", - ")", - "=C", - "10", - "+1" + "=IF(A10 > 0, A10, 0)", + "=SUM(A10, B10)", + "=C10+1" ], [ "5", - "=IF(A", - "11", - " > 0, A", - "11", - ", 0)", - "=SUM(A", - "11", - ", B", - "11", - ")", - "=C", - "11", - "+1" + "=IF(A11 > 0, A11, 0)", + "=SUM(A11, B11)", + "=C11+1" ], [ "6", - "=IF(A", - "12", - " > 0, A", - "12", - ", 0)", - "=SUM(A", - "12", - ", B", - "12", - ")", - "=C", - "12", - "+1" + "=IF(A12 > 0, A12, 0)", + "=SUM(A12, B12)", + "=C12+1" ], [ "7", - "=IF(A", - "13", - " > 0, A", - "13", - ", 0)", - "=SUM(A", - "13", - ", B", - "13", - ")", - "=C", - "13", - "+1" + "=IF(A13 > 0, A13, 0)", + "=SUM(A13, B13)", + "=C13+1" ], [ "8", - "=IF(A", - "14", - " > 0, A", - "14", - ", 0)", - "=SUM(A", - "14", - ", B", - "14", - ")", - "=C", - "14", - "+1" + "=IF(A14 > 0, A14, 0)", + "=SUM(A14, B14)", + "=C14+1" ], [ "9", - "=IF(A", - "15", - " > 0, A", - "15", - ", 0)", - "=SUM(A", - "15", - ", B", - "15", - ")", - "=C", - "15", - "+1" + "=IF(A15 > 0, A15, 0)", + "=SUM(A15, B15)", + "=C15+1" ] ] \ No newline at end of file diff --git a/add-in/src/api/pids/test/test2.json b/add-in/src/api/pids/test/test2.json new file mode 100644 index 0000000..31bce29 --- /dev/null +++ b/add-in/src/api/pids/test/test2.json @@ -0,0 +1,242 @@ +[ + [ + "-5", + "=IF(A", + "1", + " > 0, A", + "1", + ", 0)", + "=SUM(A", + "1", + ", B", + "1", + ")", + "=C", + "1", + "+1" + ], + [ + "-4", + "=IF(A", + "2", + " > 0, A", + "2", + ", 0)", + "=SUM(A", + "2", + ", B", + "2", + ")", + "=C", + "2", + "+1" + ], + [ + "-3", + "=IF(A", + "3", + " > 0, A", + "3", + ", 0)", + "=SUM(A", + "3", + ", B", + "3", + ")", + "=C", + "3", + "+1" + ], + [ + "-2", + "=IF(A", + "4", + " > 0, A", + "4", + ", 0)", + "=SUM(A", + "4", + ", B", + "4", + ")", + "=C", + "4", + "+1" + ], + [ + "-1", + "=IF(A", + "5", + " > 0, A", + "5", + ", 0)", + "=SUM(A", + "5", + ", B", + "5", + ")", + "=C", + "5", + "+1" + ], + [ + "0", + "=IF(A", + "6", + " > 0, A", + "6", + ", 0)", + "=SUM(A", + "6", + ", B", + "6", + ")", + "=C", + "6", + "+1" + ], + [ + "1", + "=IF(A", + "7", + " > 0, A", + "7", + ", 0)", + "=SUM(A", + "7", + ", B", + "7", + ")", + "=C", + "7", + "+1" + ], + [ + "2", + "=IF(A", + "8", + " > 0, A", + "8", + ", 0)", + "=SUM(A", + "8", + ", B", + "8", + ")", + "=C", + "8", + "+1" + ], + [ + "3", + "=IF(A", + "9", + " > 0, A", + "9", + ", 0)", + "=SUM(A", + "9", + ", B", + "9", + ")", + "=C", + "9", + "+1" + ], + [ + "4", + "=IF(A", + "10", + " > 0, A", + "10", + ", 0)", + "=SUM(A", + "10", + ", B", + "10", + ")", + "=C", + "10", + "+1" + ], + [ + "5", + "=IF(A", + "11", + " > 0, A", + "11", + ", 0)", + "=SUM(A", + "11", + ", B", + "11", + ")", + "=C", + "11", + "+1" + ], + [ + "6", + "=IF(A", + "12", + " > 0, A", + "12", + ", 0)", + "=SUM(A", + "12", + ", B", + "12", + ")", + "=C", + "12", + "+1" + ], + [ + "7", + "=IF(A", + "13", + " > 0, A", + "13", + ", 0)", + "=SUM(A", + "13", + ", B", + "13", + ")", + "=C", + "13", + "+1" + ], + [ + "8", + "=IF(A", + "14", + " > 0, A", + "14", + ", 0)", + "=SUM(A", + "14", + ", B", + "14", + ")", + "=C", + "14", + "+1" + ], + [ + "9", + "=IF(A", + "15", + " > 0, A", + "15", + ", 0)", + "=SUM(A", + "15", + ", B", + "15", + ")", + "=C", + "15", + "+1" + ] +] \ No newline at end of file diff --git a/add-in/src/api/pids/test/test3.json b/add-in/src/api/pids/test/test3.json new file mode 100644 index 0000000..0e3a391 --- /dev/null +++ b/add-in/src/api/pids/test/test3.json @@ -0,0 +1,527 @@ +[ + [ + "-5", + "=", + "IF", + "(", + "A", + "1", + " ", + ">", + " ", + "0", + ",", + " ", + "A", + "1", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A", + "1", + ",", + " ", + "B", + "1", + ")", + "=", + "C", + "1", + "+", + "1" + ], + [ + "-4", + "=", + "IF", + "(", + "A", + "2", + " ", + ">", + " ", + "0", + ",", + " ", + "A", + "2", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A", + "2", + ",", + " ", + "B", + "2", + ")", + "=", + "C", + "2", + "+", + "1" + ], + [ + "-3", + "=", + "IF", + "(", + "A", + "3", + " ", + ">", + " ", + "0", + ",", + " ", + "A", + "3", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A", + "3", + ",", + " ", + "B", + "3", + ")", + "=", + "C", + "3", + "+", + "1" + ], + [ + "-2", + "=", + "IF", + "(", + "A", + "4", + " ", + ">", + " ", + "0", + ",", + " ", + "A", + "4", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A", + "4", + ",", + " ", + "B", + "4", + ")", + "=", + "C", + "4", + "+", + "1" + ], + [ + "-1", + "=", + "IF", + "(", + "A", + "5", + " ", + ">", + " ", + "0", + ",", + " ", + "A", + "5", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A", + "5", + ",", + " ", + "B", + "5", + ")", + "=", + "C", + "5", + "+", + "1" + ], + [ + "0", + "=", + "IF", + "(", + "A", + "6", + " ", + ">", + " ", + "0", + ",", + " ", + "A", + "6", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A", + "6", + ",", + " ", + "B", + "6", + ")", + "=", + "C", + "6", + "+", + "1" + ], + [ + "1", + "=", + "IF", + "(", + "A", + "7", + " ", + ">", + " ", + "0", + ",", + " ", + "A", + "7", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A", + "7", + ",", + " ", + "B", + "7", + ")", + "=", + "C", + "7", + "+", + "1" + ], + [ + "2", + "=", + "IF", + "(", + "A", + "8", + " ", + ">", + " ", + "0", + ",", + " ", + "A", + "8", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A", + "8", + ",", + " ", + "B", + "8", + ")", + "=", + "C", + "8", + "+", + "1" + ], + [ + "3", + "=", + "IF", + "(", + "A", + "9", + " ", + ">", + " ", + "0", + ",", + " ", + "A", + "9", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A", + "9", + ",", + " ", + "B", + "9", + ")", + "=", + "C", + "9", + "+", + "1" + ], + [ + "4", + "=", + "IF", + "(", + "A", + "10", + " ", + ">", + " ", + "0", + ",", + " ", + "A", + "10", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A", + "10", + ",", + " ", + "B", + "10", + ")", + "=", + "C", + "10", + "+", + "1" + ], + [ + "5", + "=", + "IF", + "(", + "A", + "11", + " ", + ">", + " ", + "0", + ",", + " ", + "A", + "11", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A", + "11", + ",", + " ", + "B", + "11", + ")", + "=", + "C", + "11", + "+", + "1" + ], + [ + "6", + "=", + "IF", + "(", + "A", + "12", + " ", + ">", + " ", + "0", + ",", + " ", + "A", + "12", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A", + "12", + ",", + " ", + "B", + "12", + ")", + "=", + "C", + "12", + "+", + "1" + ], + [ + "7", + "=", + "IF", + "(", + "A", + "13", + " ", + ">", + " ", + "0", + ",", + " ", + "A", + "13", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A", + "13", + ",", + " ", + "B", + "13", + ")", + "=", + "C", + "13", + "+", + "1" + ], + [ + "8", + "=", + "IF", + "(", + "A", + "14", + " ", + ">", + " ", + "0", + ",", + " ", + "A", + "14", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A", + "14", + ",", + " ", + "B", + "14", + ")", + "=", + "C", + "14", + "+", + "1" + ], + [ + "9", + "=", + "IF", + "(", + "A", + "15", + " ", + ">", + " ", + "0", + ",", + " ", + "A", + "15", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A", + "15", + ",", + " ", + "B", + "15", + ")", + "=", + "C", + "15", + "+", + "1" + ] +] \ No newline at end of file diff --git a/add-in/src/api/pids/test/test4.json b/add-in/src/api/pids/test/test4.json new file mode 100644 index 0000000..2421c98 --- /dev/null +++ b/add-in/src/api/pids/test/test4.json @@ -0,0 +1,452 @@ +[ + [ + "-5", + "=", + "IF", + "(", + "A1", + " ", + ">", + " ", + "0", + ",", + " ", + "A1", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A1", + ",", + " ", + "B1", + ")", + "=", + "C1", + "+", + "1" + ], + [ + "-4", + "=", + "IF", + "(", + "A2", + " ", + ">", + " ", + "0", + ",", + " ", + "A2", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A2", + ",", + " ", + "B2", + ")", + "=", + "C2", + "+", + "1" + ], + [ + "-3", + "=", + "IF", + "(", + "A3", + " ", + ">", + " ", + "0", + ",", + " ", + "A3", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A3", + ",", + " ", + "B3", + ")", + "=", + "C3", + "+", + "1" + ], + [ + "-2", + "=", + "IF", + "(", + "A4", + " ", + ">", + " ", + "0", + ",", + " ", + "A4", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A4", + ",", + " ", + "B4", + ")", + "=", + "C4", + "+", + "1" + ], + [ + "-1", + "=", + "IF", + "(", + "A5", + " ", + ">", + " ", + "0", + ",", + " ", + "A5", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A5", + ",", + " ", + "B5", + ")", + "=", + "C5", + "+", + "1" + ], + [ + "0", + "=", + "IF", + "(", + "A6", + " ", + ">", + " ", + "0", + ",", + " ", + "A6", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A6", + ",", + " ", + "B6", + ")", + "=", + "C6", + "+", + "1" + ], + [ + "1", + "=", + "IF", + "(", + "A7", + " ", + ">", + " ", + "0", + ",", + " ", + "A7", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A7", + ",", + " ", + "B7", + ")", + "=", + "C7", + "+", + "1" + ], + [ + "2", + "=", + "IF", + "(", + "A8", + " ", + ">", + " ", + "0", + ",", + " ", + "A8", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A8", + ",", + " ", + "B8", + ")", + "=", + "C8", + "+", + "1" + ], + [ + "3", + "=", + "IF", + "(", + "A9", + " ", + ">", + " ", + "0", + ",", + " ", + "A9", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A9", + ",", + " ", + "B9", + ")", + "=", + "C9", + "+", + "1" + ], + [ + "4", + "=", + "IF", + "(", + "A10", + " ", + ">", + " ", + "0", + ",", + " ", + "A10", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A10", + ",", + " ", + "B10", + ")", + "=", + "C10", + "+", + "1" + ], + [ + "5", + "=", + "IF", + "(", + "A11", + " ", + ">", + " ", + "0", + ",", + " ", + "A11", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A11", + ",", + " ", + "B11", + ")", + "=", + "C11", + "+", + "1" + ], + [ + "6", + "=", + "IF", + "(", + "A12", + " ", + ">", + " ", + "0", + ",", + " ", + "A12", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A12", + ",", + " ", + "B12", + ")", + "=", + "C12", + "+", + "1" + ], + [ + "7", + "=", + "IF", + "(", + "A13", + " ", + ">", + " ", + "0", + ",", + " ", + "A13", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A13", + ",", + " ", + "B13", + ")", + "=", + "C13", + "+", + "1" + ], + [ + "8", + "=", + "IF", + "(", + "A14", + " ", + ">", + " ", + "0", + ",", + " ", + "A14", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A14", + ",", + " ", + "B14", + ")", + "=", + "C14", + "+", + "1" + ], + [ + "9", + "=", + "IF", + "(", + "A15", + " ", + ">", + " ", + "0", + ",", + " ", + "A15", + ",", + " ", + "0", + ")", + "=", + "SUM", + "(", + "A15", + ",", + " ", + "B15", + ")", + "=", + "C15", + "+", + "1" + ] +] \ No newline at end of file diff --git a/add-in/src/api/pids/test/test5.json b/add-in/src/api/pids/test/test5.json new file mode 100644 index 0000000..19d16c4 --- /dev/null +++ b/add-in/src/api/pids/test/test5.json @@ -0,0 +1,172 @@ +[ + [ + "9755", + ":", + "FE45", + ":", + "E943", + ":", + "6B7F", + ":", + "75D6", + ":", + "8BC5", + ":", + "F0AD", + ":", + "BCBC" + ], + [ + "5464", + ":", + "F26A", + ":", + "4EC5", + ":", + "5559", + ":", + "43D0", + ":", + "9710", + ":", + "859F", + ":", + "AD3D" + ], + [ + "DF4F", + ":", + "61CB", + ":", + "1A3A", + ":", + "721D", + ":", + "B0C2", + ":", + "04AD", + ":", + "CDA9", + ":", + "1188" + ], + [ + "7DAE", + ":", + "0162", + ":", + "C620", + ":", + "716C", + ":", + "C25E", + ":", + "6E58", + ":", + "CEB8", + ":", + "194C" + ], + [ + "759C", + ":", + "1B54", + ":", + "42CB", + ":", + "C5A2", + ":", + "DD0A", + ":", + "6F3C", + ":", + "76B4", + ":", + "A3BD" + ], + [ + "4222", + ":", + "B2D9", + ":", + "5769", + ":", + "00F3", + ":", + "9C20", + ":", + "F83F", + ":", + "67C0", + ":", + "469B" + ], + [ + "89BA", + ":", + "F2F0", + ":", + "2CD9", + ":", + "320C", + ":", + "3DAD", + ":", + "A69E", + ":", + "16B6", + ":", + "6D00" + ], + [ + "A69D", + ":", + "046D", + ":", + "8FD4", + ":", + "2E2F", + ":", + "B111", + ":", + "C305", + ":", + "6619", + ":", + "8734" + ], + [ + "C8ED", + ":", + "72D8", + ":", + "54F0", + ":", + "F9AD", + ":", + "4A75", + ":", + "A09F", + ":", + "D179", + ":", + "9542" + ], + [ + "A3A3", + ":", + "DCC2", + ":", + "C369", + ":", + "886A", + ":", + "A2C3", + ":", + "F5F8", + ":", + "652F", + ":", + "09B5" + ] +] \ No newline at end of file diff --git a/add-in/src/api/pids/test/test_notebook.ipynb b/add-in/src/api/pids/test/test_notebook.ipynb index 91f7c8d..fa4ffb4 100644 --- a/add-in/src/api/pids/test/test_notebook.ipynb +++ b/add-in/src/api/pids/test/test_notebook.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "id": "ae79869c", "metadata": {}, "outputs": [], @@ -13,14 +13,14 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 2, "id": "9ad77165", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e8b6a9f824cf43f9b69cd67232974532", + "model_id": "e026459fc2844bc08081fcd81b80221f", "version_major": 2, "version_minor": 0 }, @@ -34,7 +34,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "954c8f3cb46243fbae07819ce4130698", + "model_id": "4d662f9b1c9f447d9b6ec8fb3ece17fa", "version_major": 2, "version_minor": 0 }, @@ -53,9 +53,169 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "f41c4271", "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8cfbe84350f348f99eb34d3ce2a2e829", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Button(description='Toggle Pandas/Lux', layout=Layout(top='5px', width='140px'), style=ButtonStyle())" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ce9b9ae4ef6e4351a55f7e95c876afc1", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "df2 = pd.read_json(\"test2.json\")\n", + "df2" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "dedaaf9a", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "bdda23876e1f4c23a6e8745503751643", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Button(description='Toggle Pandas/Lux', layout=Layout(top='5px', width='140px'), style=ButtonStyle())" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a765a668a49743eba501b63f39250fc6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "df3 = pd.read_json(\"test3.json\")\n", + "df3" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "2373cf4c", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d650149c6a5e46e6809a7ab5a46ed9af", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Button(description='Toggle Pandas/Lux', layout=Layout(top='5px', width='140px'), style=ButtonStyle())" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7671a2011dcb435c99fb822030336d0d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "df4 = pd.read_json(\"test4.json\")\n", + "df4" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "c5bccaac", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2a6f00f25e604cfa9bf0050efb185e39", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Button(description='Toggle Pandas/Lux', layout=Layout(top='5px', width='140px'), style=ButtonStyle())" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ac0d42e8b99c465b879069eb60f36d9f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "df5 = pd.read_json(\"test5.json\")\n", + "df5" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a184a6fb", + "metadata": {}, "outputs": [], "source": [] }