From 13670d3ec80d15b23429c2eb1c5f018e31213fd3 Mon Sep 17 00:00:00 2001 From: Alfu Date: Fri, 31 May 2024 11:53:05 +0200 Subject: [PATCH 1/7] wip winston logging --- package-lock.json | 579 +++++++++++++++++++++++++++++++- package.json | 4 +- src/ECGNode.ts | 33 +- src/datafetch/ECGDataFetcher.ts | 4 +- src/utils/Logger.ts | 47 ++- 5 files changed, 647 insertions(+), 20 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1740639..41751ab 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,9 @@ "simple-statistics": "^7.8.3", "swagger-jsdoc": "^6.2.8", "swagger-ui-express": "^5.0.0", - "utf-8-validate": "^5.0.10" + "utf-8-validate": "^5.0.10", + "winston": "^3.13.0", + "winston-loki": "^6.1.2" }, "devDependencies": { "@typechain/ethers-v6": "^0.5.1", @@ -114,6 +116,14 @@ "integrity": "sha512-iZf+UWfL+DogJVpd/xMQyP6X6McYd6ArdYoPMiv/zlOTzeXXfQbYxBNJJBF6tThvsjLMbA8tLjkCdm9RWMFCCw==", "dev": true }, + "node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "engines": { + "node": ">=0.1.90" + } + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -126,6 +136,16 @@ "node": ">=12" } }, + "node_modules/@dabh/diagnostics": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", + "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", + "dependencies": { + "colorspace": "1.1.x", + "enabled": "2.0.x", + "kuler": "^2.0.0" + } + }, "node_modules/@discordjs/builders": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/@discordjs/builders/-/builders-1.7.0.tgz", @@ -386,6 +406,201 @@ "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz", "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==" }, + "node_modules/@napi-rs/snappy-android-arm-eabi": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm-eabi/-/snappy-android-arm-eabi-7.2.2.tgz", + "integrity": "sha512-H7DuVkPCK5BlAr1NfSU8bDEN7gYs+R78pSHhDng83QxRnCLmVIZk33ymmIwurmoA1HrdTxbkbuNl+lMvNqnytw==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-android-arm64": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm64/-/snappy-android-arm64-7.2.2.tgz", + "integrity": "sha512-2R/A3qok+nGtpVK8oUMcrIi5OMDckGYNoBLFyli3zp8w6IArPRfg1yOfVUcHvpUDTo9T7LOS1fXgMOoC796eQw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-darwin-arm64": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-arm64/-/snappy-darwin-arm64-7.2.2.tgz", + "integrity": "sha512-USgArHbfrmdbuq33bD5ssbkPIoT7YCXCRLmZpDS6dMDrx+iM7eD2BecNbOOo7/v1eu6TRmQ0xOzeQ6I/9FIi5g==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-darwin-x64": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-x64/-/snappy-darwin-x64-7.2.2.tgz", + "integrity": "sha512-0APDu8iO5iT0IJKblk2lH0VpWSl9zOZndZKnBYIc+ei1npw2L5QvuErFOTeTdHBtzvUHASB+9bvgaWnQo4PvTQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-freebsd-x64": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-freebsd-x64/-/snappy-freebsd-x64-7.2.2.tgz", + "integrity": "sha512-mRTCJsuzy0o/B0Hnp9CwNB5V6cOJ4wedDTWEthsdKHSsQlO7WU9W1yP7H3Qv3Ccp/ZfMyrmG98Ad7u7lG58WXA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-arm-gnueabihf": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm-gnueabihf/-/snappy-linux-arm-gnueabihf-7.2.2.tgz", + "integrity": "sha512-v1uzm8+6uYjasBPcFkv90VLZ+WhLzr/tnfkZ/iD9mHYiULqkqpRuC8zvc3FZaJy5wLQE9zTDkTJN1IvUcZ+Vcg==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-arm64-gnu": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-gnu/-/snappy-linux-arm64-gnu-7.2.2.tgz", + "integrity": "sha512-LrEMa5pBScs4GXWOn6ZYXfQ72IzoolZw5txqUHVGs8eK4g1HR9HTHhb2oY5ySNaKakG5sOgMsb1rwaEnjhChmQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-arm64-musl": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-musl/-/snappy-linux-arm64-musl-7.2.2.tgz", + "integrity": "sha512-3orWZo9hUpGQcB+3aTLW7UFDqNCQfbr0+MvV67x8nMNYj5eAeUtMmUE/HxLznHO4eZ1qSqiTwLbVx05/Socdlw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-x64-gnu": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-gnu/-/snappy-linux-x64-gnu-7.2.2.tgz", + "integrity": "sha512-jZt8Jit/HHDcavt80zxEkDpH+R1Ic0ssiVCoueASzMXa7vwPJeF4ZxZyqUw4qeSy7n8UUExomu8G8ZbP6VKhgw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-x64-musl": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-musl/-/snappy-linux-x64-musl-7.2.2.tgz", + "integrity": "sha512-Dh96IXgcZrV39a+Tej/owcd9vr5ihiZ3KRix11rr1v0MWtVb61+H1GXXlz6+Zcx9y8jM1NmOuiIuJwkV4vZ4WA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-win32-arm64-msvc": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-arm64-msvc/-/snappy-win32-arm64-msvc-7.2.2.tgz", + "integrity": "sha512-9No0b3xGbHSWv2wtLEn3MO76Yopn1U2TdemZpCaEgOGccz1V+a/1d16Piz3ofSmnA13HGFz3h9NwZH9EOaIgYA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-win32-ia32-msvc": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-ia32-msvc/-/snappy-win32-ia32-msvc-7.2.2.tgz", + "integrity": "sha512-QiGe+0G86J74Qz1JcHtBwM3OYdTni1hX1PFyLRo3HhQUSpmi13Bzc1En7APn+6Pvo7gkrcy81dObGLDSxFAkQQ==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-win32-x64-msvc": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-x64-msvc/-/snappy-win32-x64-msvc-7.2.2.tgz", + "integrity": "sha512-a43cyx1nK0daw6BZxVcvDEXxKMFLSBSDTAhsFD0VqSKcC7MGUBMaqyoWUcMiI7LBSz4bxUmxDWKfCYzpEmeb3w==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@noble/curves": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.2.0.tgz", @@ -443,6 +658,60 @@ "node": ">= 8" } }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" + }, "node_modules/@sapphire/async-queue": { "version": "1.5.2", "resolved": "https://registry.npmjs.org/@sapphire/async-queue/-/async-queue-1.5.2.tgz", @@ -659,6 +928,11 @@ "@types/serve-static": "*" } }, + "node_modules/@types/triple-beam": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.5.tgz", + "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==" + }, "node_modules/@types/ws": { "version": "8.5.10", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.10.tgz", @@ -1004,6 +1278,19 @@ "node": ">=8" } }, + "node_modules/async": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", + "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==" + }, + "node_modules/async-exit-hook": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/async-exit-hook/-/async-exit-hook-2.0.1.tgz", + "integrity": "sha512-NW2cX8m1Q7KPA7a5M2ULQeZ2wR5qI5PAbw5L0UOMxdioVk9PMZ0h1TmyZEkPYrCvYjDlFICusOu1dlEKAAeXBw==", + "engines": { + "node": ">=0.12.0" + } + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -1098,6 +1385,17 @@ "node": ">=8" } }, + "node_modules/btoa": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz", + "integrity": "sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g==", + "bin": { + "btoa": "bin/btoa.js" + }, + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/bufferutil": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/bufferutil/-/bufferutil-4.0.8.tgz", @@ -1205,6 +1503,15 @@ "node": ">= 6" } }, + "node_modules/color": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", + "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==", + "dependencies": { + "color-convert": "^1.9.3", + "color-string": "^1.6.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -1220,8 +1527,38 @@ "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "node_modules/color/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "node_modules/colorspace": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz", + "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==", + "dependencies": { + "color": "^3.1.3", + "text-hex": "1.0.x" + } }, "node_modules/combined-stream": { "version": "1.0.8", @@ -1686,6 +2023,11 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, + "node_modules/enabled": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", + "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" + }, "node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -2110,6 +2452,11 @@ "reusify": "^1.0.4" } }, + "node_modules/fecha": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", + "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==" + }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -2247,6 +2594,11 @@ "integrity": "sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ==", "dev": true }, + "node_modules/fn.name": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", + "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" + }, "node_modules/follow-redirects": { "version": "1.15.6", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", @@ -2609,6 +2961,11 @@ "node": ">= 0.10" } }, + "node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + }, "node_modules/is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -2660,6 +3017,17 @@ "node": ">=8" } }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -2719,6 +3087,11 @@ "json-buffer": "3.0.1" } }, + "node_modules/kuler": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", + "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -2784,6 +3157,27 @@ "resolved": "https://registry.npmjs.org/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz", "integrity": "sha512-QZ1d4xoBHYUeuouhEq3lk3Uq7ldgyFXGBhg04+oRLnIz8o9T65Eh+8YdroUwn846zchkA9yDsDl5CVVaV2nqYw==" }, + "node_modules/logform": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.6.0.tgz", + "integrity": "sha512-1ulHeNPp6k/LD8H91o7VYFBng5i1BDE7HoKxVbZiGFidS1Rj65qcywLxX+pVfAPoQJEjRdvKcusKwOupHCVOVQ==", + "dependencies": { + "@colors/colors": "1.6.0", + "@types/triple-beam": "^1.3.2", + "fecha": "^4.2.0", + "ms": "^2.1.1", + "safe-stable-stringify": "^2.3.1", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/long": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" + }, "node_modules/lru-cache": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.1.0.tgz", @@ -2912,8 +3306,7 @@ "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/natural-compare": { "version": "1.4.0", @@ -3034,6 +3427,14 @@ "onchange": "dist/bin.js" } }, + "node_modules/one-time": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", + "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", + "dependencies": { + "fn.name": "1.x.x" + } + }, "node_modules/openapi-types": { "version": "12.1.3", "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", @@ -3220,6 +3621,29 @@ "node": ">=6.0.0" } }, + "node_modules/protobufjs": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.3.0.tgz", + "integrity": "sha512-YWD03n3shzV9ImZRX3ccbjqLxj7NokGN0V/ESiBV5xWqrommYHYiihuIyavq03pWSGqlyvYUFmfoMKd+1rPA/g==", + "hasInstallScript": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -3302,6 +3726,19 @@ "node": ">= 0.8" } }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -3412,6 +3849,14 @@ } ] }, + "node_modules/safe-stable-stringify": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz", + "integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==", + "engines": { + "node": ">=10" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -3566,6 +4011,14 @@ "node": "*" } }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -3575,6 +4028,42 @@ "node": ">=8" } }, + "node_modules/snappy": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/snappy/-/snappy-7.2.2.tgz", + "integrity": "sha512-iADMq1kY0v3vJmGTuKcFWSXt15qYUz7wFkArOrsSg0IFfI3nJqIJvK2/ZbEIndg7erIJLtAVX2nSOqPz7DcwbA==", + "optional": true, + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/snappy-android-arm-eabi": "7.2.2", + "@napi-rs/snappy-android-arm64": "7.2.2", + "@napi-rs/snappy-darwin-arm64": "7.2.2", + "@napi-rs/snappy-darwin-x64": "7.2.2", + "@napi-rs/snappy-freebsd-x64": "7.2.2", + "@napi-rs/snappy-linux-arm-gnueabihf": "7.2.2", + "@napi-rs/snappy-linux-arm64-gnu": "7.2.2", + "@napi-rs/snappy-linux-arm64-musl": "7.2.2", + "@napi-rs/snappy-linux-x64-gnu": "7.2.2", + "@napi-rs/snappy-linux-x64-musl": "7.2.2", + "@napi-rs/snappy-win32-arm64-msvc": "7.2.2", + "@napi-rs/snappy-win32-ia32-msvc": "7.2.2", + "@napi-rs/snappy-win32-x64-msvc": "7.2.2" + } + }, + "node_modules/stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", + "engines": { + "node": "*" + } + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -3583,6 +4072,14 @@ "node": ">= 0.8" } }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string-format": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/string-format/-/string-format-2.0.0.tgz", @@ -3726,6 +4223,11 @@ "node": ">=8" } }, + "node_modules/text-hex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -3766,6 +4268,14 @@ "tree-kill": "cli.js" } }, + "node_modules/triple-beam": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", + "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==", + "engines": { + "node": ">= 14.0.0" + } + }, "node_modules/ts-command-line-args": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/ts-command-line-args/-/ts-command-line-args-2.5.1.tgz", @@ -4008,6 +4518,11 @@ "punycode": "^2.1.0" } }, + "node_modules/url-polyfill": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/url-polyfill/-/url-polyfill-1.1.12.tgz", + "integrity": "sha512-mYFmBHCapZjtcNHW0MDq9967t+z4Dmg5CJ0KqysK3+ZbyoNOWQHksGCTWwDhxGXllkWlOc10Xfko6v4a3ucM6A==" + }, "node_modules/utf-8-validate": { "version": "5.0.10", "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz", @@ -4020,6 +4535,11 @@ "node": ">=6.14.2" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -4079,6 +4599,55 @@ "node": ">= 8" } }, + "node_modules/winston": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.13.0.tgz", + "integrity": "sha512-rwidmA1w3SE4j0E5MuIufFhyJPBDG7Nu71RkZor1p2+qHvJSZ9GYDA81AyleQcZbh/+V6HjeBdfnTZJm9rSeQQ==", + "dependencies": { + "@colors/colors": "^1.6.0", + "@dabh/diagnostics": "^2.0.2", + "async": "^3.2.3", + "is-stream": "^2.0.0", + "logform": "^2.4.0", + "one-time": "^1.0.0", + "readable-stream": "^3.4.0", + "safe-stable-stringify": "^2.3.1", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.7.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/winston-loki": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/winston-loki/-/winston-loki-6.1.2.tgz", + "integrity": "sha512-l1iqDDaEUt63Q8arDsiVCXIrK3jLjPOEc5UTs+WMVnWf8D+A8ZRErAAXKDOduT240aNGzpTbCwe5zfdqqLlzMg==", + "dependencies": { + "async-exit-hook": "2.0.1", + "btoa": "^1.2.1", + "protobufjs": "^7.2.4", + "url-polyfill": "^1.1.12", + "winston-transport": "^4.3.0" + }, + "optionalDependencies": { + "snappy": "^7.2.2" + } + }, + "node_modules/winston-transport": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.7.0.tgz", + "integrity": "sha512-ajBj65K5I7denzer2IYW6+2bNIVqLGDHqDw3Ow8Ohh+vdW+rv4MZ6eiDvHoKhfJFZ2auyN8byXieDDJ96ViONg==", + "dependencies": { + "logform": "^2.3.2", + "readable-stream": "^3.6.0", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, "node_modules/wordwrapjs": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-4.0.1.tgz", diff --git a/package.json b/package.json index 99edec2..766b48b 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,8 @@ "simple-statistics": "^7.8.3", "swagger-jsdoc": "^6.2.8", "swagger-ui-express": "^5.0.0", - "utf-8-validate": "^5.0.10" + "utf-8-validate": "^5.0.10", + "winston": "^3.13.0", + "winston-loki": "^6.1.2" } } diff --git a/src/ECGNode.ts b/src/ECGNode.ts index fae576e..e533ec4 100644 --- a/src/ECGNode.ts +++ b/src/ECGNode.ts @@ -8,17 +8,26 @@ import { spawn } from 'node:child_process'; import { NodeConfig } from './model/NodeConfig'; import { GetNodeConfig, sleep } from './utils/Utils'; import * as dotenv from 'dotenv'; -import { Log } from './utils/Logger'; +import logger, { Log } from './utils/Logger'; import { LoadConfiguration } from './config/Config'; dotenv.config(); async function main() { process.title = 'ECG_NODE'; - Log(`[ECG-NODE] STARTED FOR MARKET_ID: ${MARKET_ID}`); + logger.info(`[ECG-NODE] STARTED FOR MARKET_ID: ${MARKET_ID}`); + logger.debug(`[ECG-NODE] STARTED FOR MARKET_ID: ${MARKET_ID}`); if (!fs.existsSync(path.join(DATA_DIR))) { fs.mkdirSync(path.join(DATA_DIR), { recursive: true }); } + await sleep(10000); + logger.info(`NEW TESTING LOKI 1`); + await sleep(10000); + logger.info(`NEW TESTING LOKI 2`); + await sleep(10000); + await sleep(10000); + await sleep(10000); + // load external config await LoadConfiguration(); @@ -47,19 +56,19 @@ function isDebug() { async function startProcessors(nodeConfig: NodeConfig) { if (nodeConfig.processors.AUCTION_BIDDER.enabled) { - startWithSpawn('AuctionBidder'); + startWithSpawn('AuctionBidder', 'ECG_NODE_AUCTION_BIDDER'); await sleep(5000); } if (nodeConfig.processors.LOAN_CALLER.enabled) { - startWithSpawn('LoanCaller'); + startWithSpawn('LoanCaller', 'ECG_NODE_LOAN_CALLER'); await sleep(5000); } if (nodeConfig.processors.TERM_OFFBOARDER.enabled) { - startWithSpawn('TermOffboarder'); + startWithSpawn('TermOffboarder', 'ECG_NODE_TERM_OFFBOARDER'); await sleep(5000); } if (nodeConfig.processors.USER_SLASHER.enabled) { - startWithSpawn('UserSlasher'); + startWithSpawn('UserSlasher', 'ECG_NODE_USER_SLASHER'); await sleep(5000); } // if (nodeConfig.processors.TERM_ONBOARDING_WATCHER.enabled) { @@ -67,23 +76,25 @@ async function startProcessors(nodeConfig: NodeConfig) { // await sleep(5000); // } if (nodeConfig.processors.TESTNET_MARKET_MAKER.enabled) { - startWithSpawn('TestnetMarketMaker'); + startWithSpawn('TestnetMarketMaker', 'ECG_NODE_TESTNET_MARKET_MAKER'); await sleep(5000); } if (nodeConfig.processors.HISTORICAL_DATA_FETCHER.enabled) { - startWithSpawn('HistoricalDataFetcher'); + startWithSpawn('HistoricalDataFetcher', 'ECG_NODE_HISTORICAL_DATA_FETCHER'); await sleep(5000); } } -function startWithSpawn(processorName: string) { +function startWithSpawn(processorName: string, appName: string) { const nodeProcessFullPath = path.join(process.cwd(), 'processors', `${processorName}.js`); Log(`Starting ${nodeProcessFullPath}`); - const child = spawn('node', [nodeProcessFullPath], { stdio: 'inherit' }); + const updatedEnv = structuredClone(process.env); + updatedEnv.APP_NAME = appName; + const child = spawn('node', [nodeProcessFullPath], { stdio: 'inherit', env: updatedEnv }); child.on('close', (code) => { Log(`Child process exited with code ${code}. Restarting after 10sec`); - setTimeout(() => startWithSpawn(processorName), 10000); + setTimeout(() => startWithSpawn(processorName, appName), 10000); }); Log(`Started ${nodeProcessFullPath}`); diff --git a/src/datafetch/ECGDataFetcher.ts b/src/datafetch/ECGDataFetcher.ts index 51baec8..ec910ab 100644 --- a/src/datafetch/ECGDataFetcher.ts +++ b/src/datafetch/ECGDataFetcher.ts @@ -6,7 +6,7 @@ import { GetDeployBlock } from '../config/Config'; import { ReadJSON, WriteJSON } from '../utils/Utils'; import { GetWeb3Provider } from '../utils/Web3Helper'; import { FileMutex } from '../utils/FileMutex'; -import { Log } from '../utils/Logger'; +import logger, { Log } from '../utils/Logger'; import { SendNotifications } from '../utils/Notifications'; import ProtocolDataFetcher from './fetchers/ProtocolDataFetcher'; import LendingTermsFetcher from './fetchers/LendingTermsFetcher'; @@ -26,7 +26,7 @@ export async function FetchECGData() { const dtStart = performance.now(); const web3Provider = GetWeb3Provider(); const currentBlock = await web3Provider.getBlockNumber(); - Log(`FetchECGData: fetching data up to block ${currentBlock}`); + logger.debug(`FetchECGData: fetching data up to block ${currentBlock}`); const syncData: SyncData = getSyncData(); Log('FetchECGData: start fetching'); diff --git a/src/utils/Logger.ts b/src/utils/Logger.ts index 7a648a0..4bfd969 100644 --- a/src/utils/Logger.ts +++ b/src/utils/Logger.ts @@ -1,4 +1,49 @@ -import { MARKET_ID } from './Constants'; +import * as dotenv from 'dotenv'; +import winston from 'winston'; +import LokiTransport from 'winston-loki'; +dotenv.config(); + +const logger = winston.createLogger({ + level: 'debug', + format: winston.format.json(), + transports: [ + new winston.transports.Console({ + format: winston.format.printf( + (log) => + `${process.env.APP_NAME} | ${process.env.MARKET_ID ? `MARKET ${process.env.MARKET_ID} | ` : ''}${ + log.level + } | ${log.message}` + ), + level: 'debug' + }) + ] +}); + +if (process.env.LOKI_URI && process.env.LOKI_LOGIN && process.env.LOKI_PWD) { + logger.add( + new LokiTransport({ + level: 'debug', + host: process.env.LOKI_URI, + format: winston.format.printf((log) => log.message), + json: true, + labels: getDefaultMetadata(), + basicAuth: `${process.env.LOKI_LOGIN}:${process.env.LOKI_PWD}`, + useWinstonMetaAsLabels: false, + batching: true + }) + ); +} + +function getDefaultMetadata() { + return { + app: process.env.APP_NAME, + market: process.env.MARKET_ID + }; +} + +export default logger; + +///// OLD CODE // eslint-disable-next-line @typescript-eslint/no-explicit-any export function Log(msg: string, ...args: any[]) { From 59f7c512964a459a75bd1ecb8045f6235e9a7d35 Mon Sep 17 00:00:00 2001 From: Alfu Date: Fri, 31 May 2024 17:22:11 +0200 Subject: [PATCH 2/7] wip --- src/ECGNode.ts | 17 ++++------------ src/utils/Logger.ts | 4 ++-- src/utils/Utils.ts | 14 ++++++------- src/utils/Web3Helper.ts | 44 ++++------------------------------------- 4 files changed, 17 insertions(+), 62 deletions(-) diff --git a/src/ECGNode.ts b/src/ECGNode.ts index e533ec4..bed23cf 100644 --- a/src/ECGNode.ts +++ b/src/ECGNode.ts @@ -8,26 +8,17 @@ import { spawn } from 'node:child_process'; import { NodeConfig } from './model/NodeConfig'; import { GetNodeConfig, sleep } from './utils/Utils'; import * as dotenv from 'dotenv'; -import logger, { Log } from './utils/Logger'; +import logger from './utils/Logger'; import { LoadConfiguration } from './config/Config'; dotenv.config(); async function main() { process.title = 'ECG_NODE'; - logger.info(`[ECG-NODE] STARTED FOR MARKET_ID: ${MARKET_ID}`); logger.debug(`[ECG-NODE] STARTED FOR MARKET_ID: ${MARKET_ID}`); if (!fs.existsSync(path.join(DATA_DIR))) { fs.mkdirSync(path.join(DATA_DIR), { recursive: true }); } - await sleep(10000); - logger.info(`NEW TESTING LOKI 1`); - await sleep(10000); - logger.info(`NEW TESTING LOKI 2`); - await sleep(10000); - await sleep(10000); - await sleep(10000); - // load external config await LoadConfiguration(); @@ -87,16 +78,16 @@ async function startProcessors(nodeConfig: NodeConfig) { function startWithSpawn(processorName: string, appName: string) { const nodeProcessFullPath = path.join(process.cwd(), 'processors', `${processorName}.js`); - Log(`Starting ${nodeProcessFullPath}`); + logger.debug(`Starting ${nodeProcessFullPath}`); const updatedEnv = structuredClone(process.env); updatedEnv.APP_NAME = appName; const child = spawn('node', [nodeProcessFullPath], { stdio: 'inherit', env: updatedEnv }); child.on('close', (code) => { - Log(`Child process exited with code ${code}. Restarting after 10sec`); + logger.debug(`Child process exited with code ${code}. Restarting after 10sec`); setTimeout(() => startWithSpawn(processorName, appName), 10000); }); - Log(`Started ${nodeProcessFullPath}`); + logger.debug(`Started ${nodeProcessFullPath}`); } main(); diff --git a/src/utils/Logger.ts b/src/utils/Logger.ts index 4bfd969..e807473 100644 --- a/src/utils/Logger.ts +++ b/src/utils/Logger.ts @@ -4,7 +4,7 @@ import LokiTransport from 'winston-loki'; dotenv.config(); const logger = winston.createLogger({ - level: 'debug', + level: 'silly', format: winston.format.json(), transports: [ new winston.transports.Console({ @@ -14,7 +14,7 @@ const logger = winston.createLogger({ log.level } | ${log.message}` ), - level: 'debug' + level: 'silly' }) ] }); diff --git a/src/utils/Utils.ts b/src/utils/Utils.ts index b3d5bf9..1de1afe 100644 --- a/src/utils/Utils.ts +++ b/src/utils/Utils.ts @@ -4,7 +4,7 @@ import { DATA_DIR, ECG_NODE_CONFIG_FULL_FILENAME, EXPLORER_URI } from './Constan import fs from 'fs'; import path from 'path'; import { ProtocolData, ProtocolDataFileStructure } from '../model/ProtocolData'; -import { Log } from './Logger'; +import logger from './Logger'; import axios from 'axios'; export function JsonBigIntReplacer(key: string, value: any) { @@ -52,7 +52,7 @@ export async function WaitUntilScheduled(startDateMs: number, runEverySec: numbe const durationSec = (now - startDateMs) / 1000; const timeToSleepSec = runEverySec - durationSec; if (timeToSleepSec > 0) { - Log(`WaitUntilScheduled: sleeping ${timeToSleepSec} seconds`); + logger.debug(`WaitUntilScheduled: sleeping ${timeToSleepSec} seconds`); await sleep(timeToSleepSec * 1000); } } @@ -79,19 +79,19 @@ export async function retry any>( return result; } catch (e) { if (currRetry >= maxTry) { - Log(`Retry ${currRetry} failed. All ${maxTry} retry attempts exhausted`); + logger.error(`Retry ${currRetry} failed. All ${maxTry} retry attempts exhausted`); throw e; } if (axios.isAxiosError(e)) { // Access to config, request, and response - Log( + logger.debug( `Retry ${currRetry} failed calling ${e.request.protocol}//${e.request.host}/${e.request.path}: ${e}. Waiting ${retryCount} second(s)` ); } else { - Log(`Retry ${currRetry} failed: ${e}. Waiting ${retryCount} second(s)`); + logger.debug(`Retry ${currRetry} failed: ${e}. Waiting ${retryCount} second(s)`); } - // Log(e); + // logger.debug(e); await sleep(incrSleepDelay * retryCount); return retry(fn, args, maxTry, incrSleepDelay, currRetry + 1); } @@ -106,7 +106,7 @@ export function GetProtocolData(): ProtocolData { const protocolDataFilename = path.join(DATA_DIR, 'protocol-data.json'); const protocolDataFile = ReadJSON(protocolDataFilename) as ProtocolDataFileStructure; - Log(`GetProtocolData: last update ${protocolDataFile.updatedHuman}`); + logger.debug(`GetProtocolData: last update ${protocolDataFile.updatedHuman}`); if (protocolDataFile.updated < Date.now() - 2 * 3600 * 1000) { throw new Error(`Protocol data outdated: ${protocolDataFile.updatedHuman}`); diff --git a/src/utils/Web3Helper.ts b/src/utils/Web3Helper.ts index f0e86a8..70a8517 100644 --- a/src/utils/Web3Helper.ts +++ b/src/utils/Web3Helper.ts @@ -10,7 +10,7 @@ import { } from 'ethers'; import { sleep } from './Utils'; import { average } from 'simple-statistics'; -import { Log } from './Logger'; +import logger from './Logger'; import { HttpPost } from './HttpHelper'; import { TokenConfig } from '../config/Config'; import { ERC20__factory } from '../contracts/types'; @@ -150,7 +150,6 @@ export async function FetchAllEventsMulti( blockStepLimit?: number ): Promise { const extractedArray: DefaultLog[] = []; - //Log(`${logPrefix}: will fetch events for ${targetBlock - startBlock + 1} blocks`); let blockStep = blockStepLimit && blockStepLimit < initBlockStep ? blockStepLimit : initBlockStep; let fromBlock = startBlock; let toBlock = 0; @@ -172,8 +171,6 @@ export async function FetchAllEventsMulti( try { events = await web3Provider.getLogs(filter); } catch (e) { - Log('multi query filter error:', e); - blockStep = Math.round(blockStep / 2); if (blockStep < 1000) { blockStep = 1000; @@ -181,7 +178,7 @@ export async function FetchAllEventsMulti( toBlock = 0; cptError++; if (cptError >= 15) { - Log(`getPastEvents error: ${e}`); + logger.error(`getPastEvents error: ${e}`); throw e; } await sleep(5000); @@ -224,12 +221,6 @@ export async function FetchAllEventsMulti( blockStep = blockStep * 2; } - /*Log( - `${logPrefix}: [${fromBlock} - ${toBlock}] found ${events.length} events after ${cptError} errors (fetched ${ - toBlock - fromBlock + 1 - } blocks). Current results: ${extractedArray.length}` - );*/ - cptError = 0; fromBlock = toBlock + 1; @@ -238,9 +229,6 @@ export async function FetchAllEventsMulti( } } - /*Log( - `${logPrefix}: found ${extractedArray.length} events in range [${startBlock} ${targetBlock}]` - );*/ return extractedArray; } @@ -254,7 +242,6 @@ export async function FetchAllEvents( ): Promise { const extractedArray: DefaultLog[] = []; - //Log(`${logPrefix}: will fetch events for ${targetBlock - startBlock + 1} blocks`); let blockStep = blockStepLimit && blockStepLimit < initBlockStep ? blockStepLimit : initBlockStep; let fromBlock = startBlock; let toBlock = 0; @@ -269,8 +256,6 @@ export async function FetchAllEvents( try { events = await contract.queryFilter(eventName, fromBlock, toBlock); } catch (e) { - Log('all query filter error:', e); - blockStep = Math.round(blockStep / 2); if (blockStep < 1000) { blockStep = 1000; @@ -278,7 +263,7 @@ export async function FetchAllEvents( toBlock = 0; cptError++; if (cptError >= 15) { - Log(`getPastEvents error: ${e}`); + logger.error(`getPastEvents error: ${e}`); throw e; } await sleep(5000); @@ -325,12 +310,6 @@ export async function FetchAllEvents( blockStep = blockStep * 2; } - /*Log( - `${logPrefix}: [${fromBlock} - ${toBlock}] found ${events.length} events after ${cptError} errors (fetched ${ - toBlock - fromBlock + 1 - } blocks). Current results: ${extractedArray.length}` - );*/ - cptError = 0; fromBlock = toBlock + 1; @@ -339,9 +318,6 @@ export async function FetchAllEvents( } } - /*Log( - `${logPrefix}: found ${extractedArray.length} events in range [${startBlock} ${targetBlock}]` - );*/ return extractedArray; } @@ -356,8 +332,6 @@ export async function FetchAllEventsAndExtractStringArray( ): Promise { const extractedArray: Set = new Set(); - const logPrefix = `${contractName}`; - //Log(`${logPrefix}: will fetch events for ${targetBlock - startBlock + 1} blocks`); let blockStep = blockStepLimit && blockStepLimit < initBlockStep ? blockStepLimit : initBlockStep; let fromBlock = startBlock; let toBlock = 0; @@ -372,7 +346,6 @@ export async function FetchAllEventsAndExtractStringArray( try { events = await contract.queryFilter(eventName, fromBlock, toBlock); } catch (e) { - Log('query filter error:', e); blockStep = Math.round(blockStep / 2); if (blockStep < 1000) { blockStep = 1000; @@ -380,7 +353,7 @@ export async function FetchAllEventsAndExtractStringArray( toBlock = 0; cptError++; if (cptError >= 15) { - Log(`getPastEvents error: ${e}`); + logger.error(`getPastEvents error: ${e}`); throw e; } await sleep(5000); @@ -415,12 +388,6 @@ export async function FetchAllEventsAndExtractStringArray( blockStep = blockStep * 2; } - // Log( - // `${logPrefix}: [${fromBlock} - ${toBlock}] found ${events.length} events after ${cptError} errors (fetched ${ - // toBlock - fromBlock + 1 - // } blocks). Current results: ${extractedArray.size}` - // ); - cptError = 0; fromBlock = toBlock + 1; @@ -429,9 +396,6 @@ export async function FetchAllEventsAndExtractStringArray( } } - /*Log( - `${logPrefix}: found ${extractedArray.size} ${argNames.join(',')} in range [${startBlock} ${targetBlock}]` - );*/ return Array.from(extractedArray); } From c7b5365fe9751c259830fe5cdfa3fc6a7f78a0d1 Mon Sep 17 00:00:00 2001 From: Alfu Date: Tue, 4 Jun 2024 09:29:26 +0200 Subject: [PATCH 3/7] use new logger everwhere --- src/api/Api.ts | 6 +- src/api/middlewares/LoggerMiddleware.ts | 4 +- src/api/routes/ProtocolDataRoutes.ts | 1 - src/datafetch/ECGDataFetcher.ts | 26 ++++---- src/datafetch/EventProcessor.ts | 29 ++++---- src/datafetch/EventWatcher.ts | 36 +++++----- src/datafetch/fetchers/AuctionsFetcher.ts | 18 ++--- src/datafetch/fetchers/GaugesFetcher.ts | 6 +- src/datafetch/fetchers/LastActivityFetcher.ts | 18 ++--- src/datafetch/fetchers/LendingTermsFetcher.ts | 20 ++---- src/datafetch/fetchers/LoansFetcher.ts | 20 +++--- .../fetchers/TermsProposalFetcher.ts | 6 +- src/processors/AuctionBidder.ts | 30 ++++----- src/processors/HistoricalDataFetcher.ts | 66 +++++++++---------- src/processors/LoanCaller.ts | 10 +-- src/processors/TermOffboarder.ts | 26 ++++---- src/processors/TermOnboardingWatcher.ts | 16 ++--- src/processors/TestnetMarketMaker.ts | 18 ++--- src/processors/UserSlasher.ts | 12 ++-- src/services/cache/CacheService.ts | 5 -- src/services/price/PriceService.ts | 58 ++++++++-------- src/spam-watcher/SpamWatcher.ts | 24 +++---- src/utils/FileMutex.ts | 4 +- src/utils/Logger.ts | 30 ++------- src/utils/TelegramHelper.ts | 14 ++-- 25 files changed, 228 insertions(+), 275 deletions(-) diff --git a/src/api/Api.ts b/src/api/Api.ts index 65226c5..50b52ba 100644 --- a/src/api/Api.ts +++ b/src/api/Api.ts @@ -11,8 +11,8 @@ import swaggerUi from 'swagger-ui-express'; import swaggerJsdoc from 'swagger-jsdoc'; import dotenv from 'dotenv'; -import { Log } from '../utils/Logger'; import { LoadTokens } from '../config/Config'; +import logger from '../utils/Logger'; dotenv.config(); const port = process.env.API_PORT || 17777; @@ -54,7 +54,7 @@ app.use('/api/markets/', marketDataRoutes); app.use('/api/protocol/', protocolDataRoutes); app.listen(port, () => { - Log(`⚡️[server]: Server is running. See doc: http://localhost:${port}/api-docs`); + logger.debug(`⚡️[server]: Server is running. See doc: http://localhost:${port}/api-docs`); }); process.on('SIGINT', cleanup); @@ -62,6 +62,6 @@ process.on('SIGTERM', cleanup); async function cleanup() { // do cleanup if needed - Log('shutdown requested'); + logger.debug('shutdown requested'); process.exit(); } diff --git a/src/api/middlewares/LoggerMiddleware.ts b/src/api/middlewares/LoggerMiddleware.ts index 29701e6..f3f247e 100644 --- a/src/api/middlewares/LoggerMiddleware.ts +++ b/src/api/middlewares/LoggerMiddleware.ts @@ -1,8 +1,8 @@ import { Request, Response, NextFunction } from 'express'; -import { Log } from '../../utils/Logger'; +import logger from '../../utils/Logger'; const loggerMiddleware = (req: Request, _: Response, next: NextFunction) => { - Log(`[${new Date().toISOString()}] ${req.method} ${req.originalUrl}`); + logger.debug(`[${new Date().toISOString()}] ${req.method} ${req.originalUrl}`); next(); }; diff --git a/src/api/routes/ProtocolDataRoutes.ts b/src/api/routes/ProtocolDataRoutes.ts index a338de7..2e05552 100644 --- a/src/api/routes/ProtocolDataRoutes.ts +++ b/src/api/routes/ProtocolDataRoutes.ts @@ -1,6 +1,5 @@ import express, { Request, Response } from 'express'; import SimpleCacheService from '../../services/cache/CacheService'; -import MarketDataController from '../controllers/MarketDataController'; import ProtocolDataController from '../controllers/ProtocolDataController'; const router = express.Router(); diff --git a/src/datafetch/ECGDataFetcher.ts b/src/datafetch/ECGDataFetcher.ts index ec910ab..e0d5020 100644 --- a/src/datafetch/ECGDataFetcher.ts +++ b/src/datafetch/ECGDataFetcher.ts @@ -6,7 +6,7 @@ import { GetDeployBlock } from '../config/Config'; import { ReadJSON, WriteJSON } from '../utils/Utils'; import { GetWeb3Provider } from '../utils/Web3Helper'; import { FileMutex } from '../utils/FileMutex'; -import logger, { Log } from '../utils/Logger'; +import logger from '../utils/Logger'; import { SendNotifications } from '../utils/Notifications'; import ProtocolDataFetcher from './fetchers/ProtocolDataFetcher'; import LendingTermsFetcher from './fetchers/LendingTermsFetcher'; @@ -29,33 +29,33 @@ export async function FetchECGData() { logger.debug(`FetchECGData: fetching data up to block ${currentBlock}`); const syncData: SyncData = getSyncData(); - Log('FetchECGData: start fetching'); + logger.debug('FetchECGData: start fetching'); let fetchStart = performance.now(); const protocolData = await ProtocolDataFetcher.fetchAndSaveProtocolData(web3Provider); - Log(`FetchECGData: protocol data took: ${(performance.now() - fetchStart).toFixed(1)} ms`); + logger.debug(`FetchECGData: protocol data took: ${(performance.now() - fetchStart).toFixed(1)} ms`); fetchStart = performance.now(); const terms = await LendingTermsFetcher.fetchAndSaveTerms(web3Provider, currentBlock); - Log(`FetchECGData: terms data took: ${(performance.now() - fetchStart).toFixed(1)} ms`); + logger.debug(`FetchECGData: terms data took: ${(performance.now() - fetchStart).toFixed(1)} ms`); fetchStart = performance.now(); const loans = await LoansFetcher.fetchAndSaveLoans(web3Provider, terms, syncData, currentBlock); - Log(`FetchECGData: loan data took: ${(performance.now() - fetchStart).toFixed(1)} ms`); + logger.debug(`FetchECGData: loan data took: ${(performance.now() - fetchStart).toFixed(1)} ms`); fetchStart = performance.now(); const gauges = await GaugesFetcher.fetchAndSaveGauges(web3Provider, syncData, currentBlock); - Log(`FetchECGData: gauges data took: ${(performance.now() - fetchStart).toFixed(1)} ms`); + logger.debug(`FetchECGData: gauges data took: ${(performance.now() - fetchStart).toFixed(1)} ms`); fetchStart = performance.now(); const auctions = await AuctionsFetcher.fetchAndSaveAuctions(web3Provider, terms, syncData, currentBlock); - Log(`FetchECGData: auctions data took: ${(performance.now() - fetchStart).toFixed(1)} ms`); + logger.debug(`FetchECGData: auctions data took: ${(performance.now() - fetchStart).toFixed(1)} ms`); fetchStart = performance.now(); const auctionsHouses = await AuctionsFetcher.fetchAndSaveAuctionHouses(web3Provider, terms); - Log(`FetchECGData: auction house data took: ${(performance.now() - fetchStart).toFixed(1)} ms`); + logger.debug(`FetchECGData: auction house data took: ${(performance.now() - fetchStart).toFixed(1)} ms`); fetchStart = performance.now(); const proposals = await TermsProposalFetcher.fetchProposals(web3Provider, syncData, currentBlock); - Log(`FetchECGData: fetchProposals data took: ${(performance.now() - fetchStart).toFixed(1)} ms`); + logger.debug(`FetchECGData: fetchProposals data took: ${(performance.now() - fetchStart).toFixed(1)} ms`); WriteJSON(path.join(DATA_DIR, 'sync.json'), syncData); const durationMs = performance.now() - dtStart; - Log(`FetchECGData: finished fetching. Fetch duration: ${durationMs.toFixed(1)} ms`); + logger.debug(`FetchECGData: finished fetching. Fetch duration: ${durationMs.toFixed(1)} ms`); } catch (e) { - Log('FetchECGData: unknown failure', e); + logger.debug('FetchECGData: unknown failure', e); lastFetch = 0; await SendNotifications('Data Fetcher', 'Unknown exception when fetching data', JSON.stringify(e)); } finally { @@ -86,9 +86,9 @@ function getSyncData() { export async function FetchIfTooOld() { if (lastFetch + SECONDS_BETWEEN_FETCHES * 1000 > Date.now()) { - Log('FetchIfTooOld: no fetch needed'); + logger.debug('FetchIfTooOld: no fetch needed'); } else { - Log('FetchIfTooOld: start fetching data'); + logger.debug('FetchIfTooOld: start fetching data'); await FetchECGData(); } } diff --git a/src/datafetch/EventProcessor.ts b/src/datafetch/EventProcessor.ts index 1e4f0ca..ba1ff3a 100644 --- a/src/datafetch/EventProcessor.ts +++ b/src/datafetch/EventProcessor.ts @@ -1,17 +1,12 @@ import { EventData, EventQueue } from '../utils/EventQueue'; import { buildTxUrl, sleep } from '../utils/Utils'; import { FetchECGData } from './ECGDataFetcher'; -import { SendNotifications, SendNotificationsSpam } from '../utils/Notifications'; -import { Log, Warn } from '../utils/Logger'; +import logger from '../utils/Logger'; import { StartEventListener } from './EventWatcher'; -import { MARKET_ID } from '../utils/Constants'; -import { GuildToken__factory, LendingTerm__factory } from '../contracts/types'; -import { GetWeb3Provider } from '../utils/Web3Helper'; -import { GetGuildTokenAddress } from '../config/Config'; let lastBlockFetched = 0; export async function StartEventProcessor() { - Log('Started the event processor'); + logger.debug('Started the event processor'); // eslint-disable-next-line no-constant-condition while (true) { @@ -21,14 +16,14 @@ export async function StartEventProcessor() { await ProcessAsync(event); } } else { - // Log('EventProcessor: sleeping'); + // logger.debug('EventProcessor: sleeping'); await sleep(1000); } } } async function ProcessAsync(event: EventData) { - Log(`NEW EVENT DETECTED AT BLOCK ${event.block}: ${event.eventName}`); + logger.debug(`NEW EVENT DETECTED AT BLOCK ${event.block}: ${event.eventName}`); if (mustUpdateProtocol(event)) { if (lastBlockFetched < event.block) { await FetchECGData(); @@ -42,7 +37,7 @@ async function ProcessAsync(event: EventData) { const msg = 'Updated backend data\n' + `Tx: ${buildTxUrl(event.txHash)}`; - Log(msg); + logger.debug(msg); } } @@ -69,7 +64,7 @@ function mustUpdateProtocol(event: EventData): boolean { function guildTokenMustUpdate(event: EventData): boolean { switch (event.eventName.toLowerCase()) { default: - Log(`GuildToken ${event.eventName} is not important`); + logger.debug(`GuildToken ${event.eventName} is not important`); return false; case 'addgauge': case 'removegauge': @@ -82,7 +77,7 @@ function guildTokenMustUpdate(event: EventData): boolean { function lendingTermMustUpdate(event: EventData): boolean { switch (event.eventName.toLowerCase()) { default: - Log(`LendingTerm ${event.eventName} is not important`); + logger.debug(`LendingTerm ${event.eventName} is not important`); return false; case 'loanopen': case 'loanaddcollateral': @@ -90,7 +85,7 @@ function lendingTermMustUpdate(event: EventData): boolean { case 'loanclose': case 'loancall': case 'setauctionhouse': - Log(`LendingTerm ${event.eventName} must force an update`); + logger.debug(`LendingTerm ${event.eventName} must force an update`); return true; } } @@ -98,10 +93,10 @@ function lendingTermMustUpdate(event: EventData): boolean { function termFactoryMustUpdate(event: EventData): boolean { switch (event.eventName.toLowerCase()) { default: - Log(`TermFactory ${event.eventName} is not important`); + logger.debug(`TermFactory ${event.eventName} is not important`); return false; case 'termcreated': - Log(`TermFactory ${event.eventName} must force an update`); + logger.debug(`TermFactory ${event.eventName} must force an update`); return true; } } @@ -109,13 +104,13 @@ function termFactoryMustUpdate(event: EventData): boolean { function onboardingMustUpdate(event: EventData): boolean { switch (event.eventName.toLowerCase()) { default: - Log(`Onboarding ${event.eventName} is not important`); + logger.debug(`Onboarding ${event.eventName} is not important`); return false; // case 'proposalexecuted': // dont check proposal executed as it will add a gauge anyway which is already fetched case 'proposalcreated': case 'proposalqueued': case 'proposalcanceled': - Log(`Onboarding ${event.eventName} must force an update`); + logger.debug(`Onboarding ${event.eventName} must force an update`); return true; } } diff --git a/src/datafetch/EventWatcher.ts b/src/datafetch/EventWatcher.ts index 5256414..9ceed2a 100644 --- a/src/datafetch/EventWatcher.ts +++ b/src/datafetch/EventWatcher.ts @@ -8,14 +8,12 @@ import LendingTermOnbardingAbi from '../contracts/abi/LendingTermOnboarding.json import { GetGuildTokenAddress, GetLendingTermFactoryAddress, GetLendingTermOnboardingAddress } from '../config/Config'; import { GuildToken__factory, LendingTermFactory__factory } from '../contracts/types'; import { GetListenerWeb3Provider } from '../utils/Web3Helper'; -import { Log } from '../utils/Logger'; -import { MulticallWrapper } from 'ethers-multicall-provider'; -import { GetGaugeForMarketId } from '../utils/ECGHelper'; import { DATA_DIR, MARKET_ID } from '../utils/Constants'; import path from 'path'; import fs from 'fs'; import { ReadJSON } from '../utils/Utils'; import { LendingTermsFileStructure } from '../model/LendingTerm'; +import logger from '../utils/Logger'; dotenv.config(); let guildTokenContract: Contract | undefined = undefined; @@ -24,7 +22,7 @@ let termFactoryContract: Contract | undefined = undefined; let termsContracts: Contract[] = []; export function StartEventListener(onlyTerms = false) { const provider = GetListenerWeb3Provider(5000); - Log(`Starting/restarting events listener, onlyTerms: ${onlyTerms}`); + logger.debug(`Starting/restarting events listener, onlyTerms: ${onlyTerms}`); if (onlyTerms) { StartLendingTermListener(provider); } else { @@ -42,9 +40,9 @@ export function StartGuildTokenListener(provider: JsonRpcProvider) { guildTokenContract.removeAllListeners(); } - Log('Started the event listener'); + logger.debug('Started the event listener'); guildTokenContract = new Contract(GetGuildTokenAddress(), GuildTokenAbi, provider); - Log(`Starting listener on guild token ${GetGuildTokenAddress()}`); + logger.debug(`Starting listener on guild token ${GetGuildTokenAddress()}`); const guildToken = GuildToken__factory.connect(GetGuildTokenAddress(), provider); const iface = new Interface(GuildTokenAbi); @@ -56,13 +54,13 @@ export function StartGuildTokenListener(provider: JsonRpcProvider) { const parsed = iface.parseLog(event.log); if (!parsed) { - Log('Could not parse event', { event }); + logger.debug('Could not parse event', { event }); return; } if (parsed.name.toLowerCase() == 'addgauge') { if (parsed.args.gaugeType && Number(parsed.args.gaugeType as bigint) != MARKET_ID) { - Log(`Event ${parsed.name} not on marketId ${MARKET_ID}, ignoring`); + logger.debug(`Event ${parsed.name} not on marketId ${MARKET_ID}, ignoring`); return; } } @@ -80,7 +78,7 @@ export function StartGuildTokenListener(provider: JsonRpcProvider) { originArgName: parsed.fragment.inputs.map((_) => _.name) }); } else { - Log(`Event ${parsed.name} not on marketId ${MARKET_ID}, ignoring`); + logger.debug(`Event ${parsed.name} not on marketId ${MARKET_ID}, ignoring`); } }); } else { @@ -101,9 +99,9 @@ export function StartOnboardingListener(provider: JsonRpcProvider) { onboardingContract.removeAllListeners(); } - Log('Started the event listener'); + logger.debug('Started the event listener'); onboardingContract = new Contract(GetLendingTermOnboardingAddress(), LendingTermOnbardingAbi, provider); - Log(`Starting listener on onboarding ${GetLendingTermOnboardingAddress()}`); + logger.debug(`Starting listener on onboarding ${GetLendingTermOnboardingAddress()}`); onboardingContract.removeAllListeners(); @@ -112,7 +110,7 @@ export function StartOnboardingListener(provider: JsonRpcProvider) { const parsed = onboardingContract?.interface.parseLog(event.log); if (!parsed) { - Log('Could not parse event', { event }); + logger.debug('Could not parse event', { event }); return; } @@ -132,9 +130,9 @@ export function StartTermFactoryListener(provider: JsonRpcProvider) { termFactoryContract.removeAllListeners(); } - Log('Started the event listener'); + logger.debug('Started the event listener'); termFactoryContract = new Contract(GetLendingTermFactoryAddress(), LendingTermFactoryAbi, provider); - Log(`Starting listener on term factory ${GetLendingTermFactoryAddress()}`); + logger.debug(`Starting listener on term factory ${GetLendingTermFactoryAddress()}`); const termFactory = LendingTermFactory__factory.connect(GetLendingTermFactoryAddress(), provider); termFactoryContract.removeAllListeners(); @@ -147,7 +145,7 @@ export function StartTermFactoryListener(provider: JsonRpcProvider) { const parsed = termFactoryContract?.interface.parseLog(event.log); if (!parsed) { - Log('Could not parse event', { event }); + logger.debug('Could not parse event', { event }); return; } @@ -168,20 +166,20 @@ export function StartLendingTermListener(provider: JsonRpcProvider) { termContract.removeAllListeners(); } termsContracts = []; - Log('Started the event listener'); + logger.debug('Started the event listener'); const termsFileName = path.join(DATA_DIR, 'terms.json'); if (!fs.existsSync(termsFileName)) { throw new Error(`Could not find file ${termsFileName}`); } const termsFile: LendingTermsFileStructure = ReadJSON(termsFileName); const termsWithDebtCeiling = termsFile.terms.filter((_) => _.debtCeiling != '0'); - Log(`Starting terms listener for ${termsWithDebtCeiling.length}/${termsFile.terms.length} terms`); + logger.debug(`Starting terms listener for ${termsWithDebtCeiling.length}/${termsFile.terms.length} terms`); // get all lending terms (gauges) from the guild token to start a listener on each one for (const lendingTermAddress of termsWithDebtCeiling.map((_) => _.termAddress)) { const termContract = new Contract(lendingTermAddress, LendingTermAbi, provider); termsContracts.push(termContract); - Log(`Starting listener on term ${lendingTermAddress}`); + logger.debug(`Starting listener on term ${lendingTermAddress}`); const iface = new Interface(LendingTermAbi); termContract.removeAllListeners(); @@ -191,7 +189,7 @@ export function StartLendingTermListener(provider: JsonRpcProvider) { const parsed = iface.parseLog(event.log); if (!parsed) { - Log('Could not parse event', { event }); + logger.debug('Could not parse event', { event }); return; } diff --git a/src/datafetch/fetchers/AuctionsFetcher.ts b/src/datafetch/fetchers/AuctionsFetcher.ts index d04d32d..38d6bd6 100644 --- a/src/datafetch/fetchers/AuctionsFetcher.ts +++ b/src/datafetch/fetchers/AuctionsFetcher.ts @@ -5,13 +5,13 @@ import { AuctionHouse, AuctionHouse__factory } from '../../contracts/types'; import { DATA_DIR } from '../../utils/Constants'; import path from 'path'; import { ReadJSON, WriteJSON } from '../../utils/Utils'; -import { Log } from '../../utils/Logger'; import { SyncData } from '../../model/SyncData'; import { FetchAllEvents, FetchAllEventsAndExtractStringArray } from '../../utils/Web3Helper'; import { Auction, AuctionStatus, AuctionsFileStructure } from '../../model/Auction'; import LendingTerm from '../../model/LendingTerm'; import { MulticallWrapper } from 'ethers-multicall-provider'; import { AuctionHouseData, AuctionHousesFileStructure } from '../../model/AuctionHouse'; +import logger from '../../utils/Logger'; export default class AuctionsFetcher { static async fetchAndSaveAuctions( @@ -20,7 +20,7 @@ export default class AuctionsFetcher { syncData: SyncData, currentBlock: number ) { - Log('FetchECGData[Auctions]: starting'); + logger.debug('FetchECGData[Auctions]: starting'); let alreadySavedAuctions: Auction[] = []; const auctionsFilePath = path.join(DATA_DIR, 'auctions.json'); if (fs.existsSync(auctionsFilePath)) { @@ -118,7 +118,7 @@ export default class AuctionsFetcher { const index = updateAuctions.auctions.findIndex((_) => _.loanId == loanId); if (index < 0) { // if not found, it means it might be on another market - Log(`Ignoring auction end for loan ${loanId}`); + logger.debug(`Ignoring auction end for loan ${loanId}`); continue; } else { updateAuctions.auctions[index].bidTxHash = txHash; @@ -131,11 +131,11 @@ export default class AuctionsFetcher { updateAuctions.updated = endDate; updateAuctions.updatedHuman = new Date(endDate).toISOString(); WriteJSON(auctionsFilePath, updateAuctions); - Log('FetchECGData[Auctions]: ending'); + logger.debug('FetchECGData[Auctions]: ending'); } static async fetchAndSaveAuctionHouses(web3Provider: JsonRpcProvider, terms: LendingTerm[]) { - Log('FetchECGData[AuctionHouse]: starting'); + logger.debug('FetchECGData[AuctionHouse]: starting'); let allAuctionHouses: AuctionHouseData[] = []; const auctionHousesFilePath = path.join(DATA_DIR, 'auction-houses.json'); if (fs.existsSync(auctionHousesFilePath)) { @@ -167,7 +167,7 @@ export default class AuctionsFetcher { }; WriteJSON(auctionHousesFilePath, auctionsFile); - Log('FetchECGData[AuctionHouse]: ending'); + logger.debug('FetchECGData[AuctionHouse]: ending'); return allAuctionHouses; } } @@ -184,9 +184,9 @@ async function fetchAuctionsInfo( promises.push(auctionHouseContract.getAuction(loansId.loanId)); } - Log(`FetchECGData[Auctions]: sending getAuction() multicall for ${allLoanIds.length} loans`); + logger.debug(`FetchECGData[Auctions]: sending getAuction() multicall for ${allLoanIds.length} loans`); await Promise.all(promises); - Log('FetchECGData[Auctions]: end multicall'); + logger.debug('FetchECGData[Auctions]: end multicall'); let cursor = 0; const allAuctions: Auction[] = []; @@ -196,7 +196,7 @@ async function fetchAuctionsInfo( const linkedLendingTerm = lendingTerms.find((_) => _.termAddress == auctionData.lendingTerm); if (!linkedLendingTerm) { // if not found, it means it might be on another market - Log(`Ignoring auction for loan ${loan.loanId} and lending term ${auctionData.lendingTerm}`); + logger.debug(`Ignoring auction for loan ${loan.loanId} and lending term ${auctionData.lendingTerm}`); continue; } diff --git a/src/datafetch/fetchers/GaugesFetcher.ts b/src/datafetch/fetchers/GaugesFetcher.ts index e4ede28..6e07a11 100644 --- a/src/datafetch/fetchers/GaugesFetcher.ts +++ b/src/datafetch/fetchers/GaugesFetcher.ts @@ -5,15 +5,15 @@ import { GuildToken__factory } from '../../contracts/types'; import { DATA_DIR, MARKET_ID } from '../../utils/Constants'; import path from 'path'; import { ReadJSON, WriteJSON } from '../../utils/Utils'; -import { Log } from '../../utils/Logger'; import { SyncData } from '../../model/SyncData'; import { FetchAllEvents } from '../../utils/Web3Helper'; import { GaugesFileStructure } from '../../model/Gauge'; import { GetGaugeForMarketId } from '../../utils/ECGHelper'; +import logger from '../../utils/Logger'; export default class GaugesFetcher { static async fetchAndSaveGauges(web3Provider: JsonRpcProvider, syncData: SyncData, currentBlock: number) { - Log('FetchECGData[Gauges]: starting'); + logger.debug('FetchECGData[Gauges]: starting'); let sinceBlock = GetDeployBlock(); if (syncData.gaugeSync) { sinceBlock = syncData.gaugeSync.lastBlockFetched + 1; @@ -104,6 +104,6 @@ export default class GaugesFetcher { }; syncData.gaugeSync.lastBlockFetched = currentBlock; - Log(`FetchECGData[Gauges]: Updated ${Object.keys(gaugesFile.gauges).length} gauges`); + logger.debug(`FetchECGData[Gauges]: Updated ${Object.keys(gaugesFile.gauges).length} gauges`); } } diff --git a/src/datafetch/fetchers/LastActivityFetcher.ts b/src/datafetch/fetchers/LastActivityFetcher.ts index 3ee29e5..990c125 100644 --- a/src/datafetch/fetchers/LastActivityFetcher.ts +++ b/src/datafetch/fetchers/LastActivityFetcher.ts @@ -1,11 +1,11 @@ import { BaseContract, JsonRpcProvider } from 'ethers'; import { ProtocolData } from '../../model/ProtocolData'; -import { ActivitySyncData, SyncData } from '../../model/SyncData'; +import { SyncData } from '../../model/SyncData'; import { LastActivity, LastActivityFileStructure } from '../../model/LastActivity'; import { BLOCK_PER_HOUR, DATA_DIR } from '../../utils/Constants'; import path from 'path'; import fs from 'fs'; -import { ReadJSON, WriteJSON, sleep } from '../../utils/Utils'; +import { ReadJSON, WriteJSON } from '../../utils/Utils'; import LendingTerm from '../../model/LendingTerm'; import { GuildGovernor__factory, @@ -27,12 +27,8 @@ import { getTokenByAddress } from '../../config/Config'; import { norm } from '../../utils/TokenUtils'; -import { Log } from '../../utils/Logger'; +import logger from '../../utils/Logger'; -//////////////////////////////////////////////////////////// -/// THIS ONE IS STARTED BY THE HISTORICAL DATA FETCHER ///// -//////////////// THIS IS A HACK FOR NOW //////////////////// -//////////////////////////////////////////////////////////// export default class LastActivityFetcher { static async fetchAndSaveActivity( syncData: SyncData, @@ -41,7 +37,7 @@ export default class LastActivityFetcher { protocolData: ProtocolData, terms: LendingTerm[] ) { - Log('FetchECGData[LastActivity]: starting'); + logger.debug('FetchECGData[LastActivity]: starting'); // read already saved activity let alreadySavedActivities: LastActivity[] = []; @@ -73,7 +69,7 @@ export default class LastActivityFetcher { updated: Date.now(), updatedHuman: new Date(Date.now()).toISOString() }; - Log(`FetchECGData[LastActivity]: saving ${activitiesToSave.length} activities`); + logger.debug(`FetchECGData[LastActivity]: saving ${activitiesToSave.length} activities`); WriteJSON(lastActivityPath, lastActivityFile); if (!syncData.activitySync) { @@ -93,7 +89,7 @@ async function getLoanActivity( currentBlock: number, terms: LendingTerm[] ): Promise { - Log('FetchECGData[LastActivity]: starting getLoanActivity'); + logger.debug('FetchECGData[LastActivity]: starting getLoanActivity'); const allLoanActivities: LastActivity[] = []; const fromBlock = syncData.activitySync @@ -112,7 +108,7 @@ async function getLoanActivity( allLoanActivities.push(...r); } - Log('FetchECGData[LastActivity]: ending getLoanActivity'); + logger.debug('FetchECGData[LastActivity]: ending getLoanActivity'); return allLoanActivities; } diff --git a/src/datafetch/fetchers/LendingTermsFetcher.ts b/src/datafetch/fetchers/LendingTermsFetcher.ts index 05180f1..c57be31 100644 --- a/src/datafetch/fetchers/LendingTermsFetcher.ts +++ b/src/datafetch/fetchers/LendingTermsFetcher.ts @@ -1,11 +1,5 @@ import { JsonRpcProvider } from 'ethers'; -import { ProtocolData } from '../../model/ProtocolData'; -import { - GetGuildTokenAddress, - GetProfitManagerAddress, - getTokenByAddress, - getTokenByAddressNoError -} from '../../config/Config'; +import { GetGuildTokenAddress, GetProfitManagerAddress, getTokenByAddressNoError } from '../../config/Config'; import { GuildToken__factory, LendingTerm as LendingTermType, @@ -19,13 +13,13 @@ import { MulticallWrapper } from 'ethers-multicall-provider'; import { GetGaugeForMarketId } from '../../utils/ECGHelper'; import LendingTerm, { LendingTermStatus, LendingTermsFileStructure } from '../../model/LendingTerm'; import { norm } from '../../utils/TokenUtils'; -import { Log, Warn } from '../../utils/Logger'; +import logger from '../../utils/Logger'; import { GetERC20Infos } from '../../utils/Web3Helper'; import { SendNotifications } from '../../utils/Notifications'; export default class LendingTermsFetcher { static async fetchAndSaveTerms(web3Provider: JsonRpcProvider, currentBlock: number) { - Log('FetchECGData[Terms]: starting'); + logger.debug('FetchECGData[Terms]: starting'); const multicallProvider = MulticallWrapper.wrap(web3Provider); const guildTokenContract = GuildToken__factory.connect(GetGuildTokenAddress(), multicallProvider); const gauges = await GetGaugeForMarketId(guildTokenContract, MARKET_ID, false); @@ -35,7 +29,7 @@ export default class LendingTermsFetcher { promises.push(profitManagerContract.minBorrow()); promises.push(guildTokenContract.totalTypeWeight(MARKET_ID)); for (const lendingTermAddress of gauges) { - // Log(`FetchECGData: adding call for on lending term ${lendingTermAddress}`); + // logger.debug(`FetchECGData: adding call for on lending term ${lendingTermAddress}`); const lendingTermContract = LendingTerm__factory.connect(lendingTermAddress, multicallProvider); promises.push(lendingTermContract.getParameters()); promises.push(lendingTermContract.issuance()); @@ -46,9 +40,9 @@ export default class LendingTermsFetcher { } // wait the promises - Log(`FetchECGData[Terms]: sending ${promises.length} multicall`); + logger.debug(`FetchECGData[Terms]: sending ${promises.length} multicall`); await Promise.all(promises); - Log('FetchECGData[Terms]: end multicall'); + logger.debug('FetchECGData[Terms]: end multicall'); const lendingTerms: LendingTerm[] = []; let cursor = 0; @@ -68,7 +62,7 @@ export default class LendingTermsFetcher { let collateralToken = getTokenByAddressNoError(termParameters.collateralToken); if (!collateralToken) { collateralToken = await GetERC20Infos(web3Provider, termParameters.collateralToken); - Warn( + logger.error( `Token ${termParameters.collateralToken} not found in config. ERC20 infos: ${collateralToken.symbol} / ${collateralToken.decimals} decimals` ); await SendNotifications( diff --git a/src/datafetch/fetchers/LoansFetcher.ts b/src/datafetch/fetchers/LoansFetcher.ts index 249160f..07eeefb 100644 --- a/src/datafetch/fetchers/LoansFetcher.ts +++ b/src/datafetch/fetchers/LoansFetcher.ts @@ -8,10 +8,10 @@ import path from 'path'; import { ReadJSON, WriteJSON, sleep } from '../../utils/Utils'; import { MulticallWrapper } from 'ethers-multicall-provider'; import LendingTerm from '../../model/LendingTerm'; -import { Log } from '../../utils/Logger'; import { SyncData } from '../../model/SyncData'; import { Loan, LoanStatus, LoansFileStructure } from '../../model/Loan'; import { FetchAllEvents, FetchAllEventsMulti } from '../../utils/Web3Helper'; +import logger from '../../utils/Logger'; export default class LoansFetcher { static async fetchAndSaveLoans( @@ -20,7 +20,7 @@ export default class LoansFetcher { syncData: SyncData, currentBlock: number ) { - Log('FetchECGData[Loans]: starting'); + logger.debug('FetchECGData[Loans]: starting'); let alreadySavedLoans: Loan[] = []; const loansFilePath = path.join(DATA_DIR, 'loans.json'); @@ -37,7 +37,7 @@ export default class LoansFetcher { }; if (terms.length == 0) { - Log('FetchECGData[Loans]: no terms to fetch'); + logger.debug('FetchECGData[Loans]: no terms to fetch'); WriteJSON(loansFilePath, updateLoans); return; } @@ -87,7 +87,7 @@ export default class LoansFetcher { updateLoans.updated = endDate; updateLoans.updatedHuman = new Date(endDate).toISOString(); WriteJSON(loansFilePath, updateLoans); - Log('FetchECGData[Loans]: ending'); + logger.debug('FetchECGData[Loans]: ending'); } } @@ -97,7 +97,7 @@ async function fetchNewLoanOpen( web3Provider: JsonRpcProvider, currentBlock: number ): Promise<{ termAddress: string; loanId: string; txHash: string }[]> { - Log('FetchECGData[Loans]: fetchNewLoanOpen starting'); + logger.debug('FetchECGData[Loans]: fetchNewLoanOpen starting'); const termContractInterface = LendingTerm__factory.createInterface(); const topics = termContractInterface.encodeFilterTopics('LoanOpen', []); let sinceBlock = GetDeployBlock(); @@ -125,7 +125,7 @@ async function fetchNewLoanOpen( }); } - Log('FetchECGData[Loans]: fetchNewLoanOpen ending'); + logger.debug('FetchECGData[Loans]: fetchNewLoanOpen ending'); return allNewLoansIds; } @@ -141,9 +141,9 @@ async function fetchLoansInfo( promises.push(lendingTermContract.getLoanDebt(loanData.loanId)); } - Log(`FetchECGData[Loans]: sending loans() multicall for ${allLoanIds.length} loans`); + logger.debug(`FetchECGData[Loans]: sending loans() multicall for ${allLoanIds.length} loans`); await Promise.all(promises); - Log('FetchECGData[Loans]: end multicall'); + logger.debug('FetchECGData[Loans]: end multicall'); let cursor = 0; const allLoans: Loan[] = []; @@ -188,7 +188,7 @@ async function fetchClosedEventsAndUpdateLoans( syncData: SyncData, currentBlock: number ) { - Log('FetchECGData[Loans]: fetchClosedEventsAndUpdateLoans starting'); + logger.debug('FetchECGData[Loans]: fetchClosedEventsAndUpdateLoans starting'); const termContractInterface = LendingTerm__factory.createInterface(); const topics = termContractInterface.encodeFilterTopics('LoanClose', []); let sinceBlock = GetDeployBlock(); @@ -217,7 +217,7 @@ async function fetchClosedEventsAndUpdateLoans( loan.debtRepaid = debtRepaid; } - Log('FetchECGData[Loans]: fetchClosedEventsAndUpdateLoans ending'); + logger.debug('FetchECGData[Loans]: fetchClosedEventsAndUpdateLoans ending'); } async function getLoanCloseEventsForTerm( syncData: SyncData, diff --git a/src/datafetch/fetchers/TermsProposalFetcher.ts b/src/datafetch/fetchers/TermsProposalFetcher.ts index 458415a..da04467 100644 --- a/src/datafetch/fetchers/TermsProposalFetcher.ts +++ b/src/datafetch/fetchers/TermsProposalFetcher.ts @@ -14,7 +14,6 @@ import { import { BLOCK_PER_HOUR, DATA_DIR, EXPLORER_URI, MARKET_ID } from '../../utils/Constants'; import { GetNodeConfig, ReadJSON, WriteJSON, roundTo } from '../../utils/Utils'; import { MulticallWrapper } from 'ethers-multicall-provider'; -import { Log } from '../../utils/Logger'; import { SyncData } from '../../model/SyncData'; import { FetchAllEvents, GetERC20Infos, GetL1Web3Provider } from '../../utils/Web3Helper'; import { Proposal, ProposalStatus, ProposalsFileStructure } from '../../model/Proposal'; @@ -22,11 +21,12 @@ import { norm } from '../../utils/TokenUtils'; import path from 'path'; import fs from 'fs'; import { SendNotificationsList } from '../../utils/Notifications'; +import logger from '../../utils/Logger'; const nodeConfig = GetNodeConfig(); export default class TermsProposalFetcher { static async fetchProposals(web3Provider: JsonRpcProvider, syncData: SyncData, currentBlock: number) { - Log('FetchECGData[Proposals]: starting'); + logger.debug('FetchECGData[Proposals]: starting'); let allProposals: Proposal[] = []; const proposalsFilePath = path.join(DATA_DIR, 'proposals.json'); @@ -93,7 +93,7 @@ export default class TermsProposalFetcher { } syncData.proposalSync.lastBlockFetched = currentBlock; - Log('FetchECGData[Proposals]: ending'); + logger.debug('FetchECGData[Proposals]: ending'); } } diff --git a/src/processors/AuctionBidder.ts b/src/processors/AuctionBidder.ts index ca494af..76287c0 100644 --- a/src/processors/AuctionBidder.ts +++ b/src/processors/AuctionBidder.ts @@ -18,13 +18,13 @@ import LendingTerm, { LendingTermsFileStructure } from '../model/LendingTerm'; import { norm } from '../utils/TokenUtils'; import { SendNotifications } from '../utils/Notifications'; import { GetAvgGasPrice, GetERC20Infos, GetWeb3Provider } from '../utils/Web3Helper'; -import { Log, Warn } from '../utils/Logger'; import { BidderSwapMode } from '../model/NodeConfig'; import { GetOpenOceanChainCodeByChainId, OpenOceanSwapQuoteResponse } from '../model/OpenOceanApi'; import { OneInchSwapResponse } from '../model/OneInchApi'; import { HttpGet } from '../utils/HttpHelper'; import BigNumber from 'bignumber.js'; import { PendleSwapResponse } from '../model/PendleApi'; +import logger from '../utils/Logger'; const RUN_EVERY_SEC = 15; let lastCall1Inch = 0; @@ -36,7 +36,7 @@ async function AuctionBidder() { // load external config await LoadConfiguration(); process.title = 'ECG_NODE_AUCTION_BIDDER'; - Log('starting'); + logger.debug('starting'); const auctionBidderConfig = GetNodeConfig().processors.AUCTION_BIDDER; const auctionsFilename = path.join(DATA_DIR, 'auctions.json'); @@ -55,7 +55,7 @@ async function AuctionBidder() { const creditMultiplier = GetProtocolData().creditMultiplier; const auctionsToCheck = auctionFileData.auctions.filter((_) => _.status == AuctionStatus.ACTIVE); - Log(`Will check ${auctionsToCheck.length} auctions`); + logger.debug(`Will check ${auctionsToCheck.length} auctions`); for (const auction of auctionsToCheck) { const term = termFileData.terms.find((_) => _.termAddress == auction.lendingTermAddress); @@ -66,11 +66,11 @@ async function AuctionBidder() { const bidDetail = await getBidDetails(auction.auctionHouseAddress, web3Provider, auction.loanId); if (bidDetail.auctionEnded) { - Log('Auction ended, will not try to bid'); + logger.debug('Auction ended, will not try to bid'); continue; } if (bidDetail.creditAsked == 0n && auctionBidderConfig.enableForgive) { - Log(`AuctionBidder[${auction.loanId}]: will forgive auction.`); + logger.debug(`AuctionBidder[${auction.loanId}]: will forgive auction.`); await processForgive(auction, web3Provider); continue; } @@ -84,7 +84,7 @@ async function AuctionBidder() { ); if (estimatedProfit >= auctionBidderConfig.minProfitPegToken) { - Log(`AuctionBidder[${auction.loanId}]: will bid on auction for estimated profit: ${estimatedProfit}`); + logger.debug(`AuctionBidder[${auction.loanId}]: will bid on auction for estimated profit: ${estimatedProfit}`); await processBid( auction, term, @@ -97,7 +97,7 @@ async function AuctionBidder() { continue; } - Log(`AuctionBidder[${auction.loanId}]: do not bid, profit too low: ${estimatedProfit}`); + logger.debug(`AuctionBidder[${auction.loanId}]: do not bid, profit too low: ${estimatedProfit}`); } await sleep(RUN_EVERY_SEC * 1000); @@ -118,7 +118,7 @@ async function getBidDetails( auctionEnded: false }; } catch (e) { - Log('getBidDetail exception:', e); + logger.debug('getBidDetail exception:', e); return { collateralReceived: -1n, creditAsked: -1n, @@ -137,7 +137,7 @@ async function checkBidProfitability( let collateralToken = getTokenByAddressNoError(term.collateralAddress); if (!collateralToken) { collateralToken = await GetERC20Infos(web3Provider, term.collateralAddress); - Warn( + logger.warn( `Token ${term.collateralAddress} not found in config. ERC20 infos: ${collateralToken.symbol} / ${collateralToken.decimals} decimals` ); } @@ -169,7 +169,7 @@ async function checkBidProfitability( const amountPegToken = norm(getSwapResult.pegTokenReceivedWei, pegToken.decimals); const creditCostInPegToken = norm((bidDetail.creditAsked * creditMultiplier) / 10n ** 18n); - Log( + logger.debug( `checkBidProfitability: bidding cost: ${creditCostInPegToken} ${pegToken.symbol}, gains: ${amountPegToken} ${ pegToken.symbol }. PnL: ${amountPegToken - creditCostInPegToken} ${pegToken.symbol}` @@ -243,10 +243,10 @@ async function getSwapPendle( `&syTokenOutAddr=${pendleConf.syTokenOut}` + '&slippage=0.05'; - Log(`pendle url: ${pendleHostedSdkUrl}`); + logger.debug(`pendle url: ${pendleHostedSdkUrl}`); const msToWait = 6000 - (Date.now() - lastCallPendle); // 1 call every 6 seconds if (msToWait > 0) { - Log(`Waiting ${msToWait} ms before calling pendle api`); + logger.debug(`Waiting ${msToWait} ms before calling pendle api`); await sleep(msToWait); } const pendleSwapResponse = await HttpGet(pendleHostedSdkUrl); @@ -283,7 +283,7 @@ async function getSwap1Inch( '&disableEstimate=true' + // disable onchain estimate otherwise it check if we have enough balance to do the swap, which is false `&excludedProtocols=${get1inchExcludedProtocols(chainCode)}`; - Log(`getSwap1Inch: ${oneInchApiUrl}`); + logger.debug(`getSwap1Inch: ${oneInchApiUrl}`); const msToWait = 1000 - (Date.now() - lastCall1Inch); if (msToWait > 0) { await sleep(msToWait); @@ -322,7 +322,7 @@ async function getSwapOpenOcean( ): Promise<{ pegTokenReceivedWei: bigint; swapData: string; routerAddress: string }> { // when calling openocena, the amount must be normalzed const collateralAmountNorm = norm(collateralReceivedWei, collateralToken.decimals).toFixed(8); - Log(`getSwapOpenOcean: amount ${collateralAmountNorm}`); + logger.debug(`getSwapOpenOcean: amount ${collateralAmountNorm}`); const chainId = (await web3Provider.getNetwork()).chainId; const chainCode = GetOpenOceanChainCodeByChainId(chainId); @@ -339,7 +339,7 @@ async function getSwapOpenOcean( `&account=${GetGatewayAddress()}` + `&disabledDexIds=${getOpenOceanExcludedProtocols(chainId)}`; - Log(`getSwapOpenOcean: ${openOceanURL}`); + logger.debug(`getSwapOpenOcean: ${openOceanURL}`); const openOceanResponse = await HttpGet(openOceanURL); diff --git a/src/processors/HistoricalDataFetcher.ts b/src/processors/HistoricalDataFetcher.ts index 85f8edd..0e9367f 100644 --- a/src/processors/HistoricalDataFetcher.ts +++ b/src/processors/HistoricalDataFetcher.ts @@ -39,11 +39,11 @@ import { import { MulticallWrapper } from 'ethers-multicall-provider'; import { Loan, LoanStatus } from '../model/Loan'; import { HistoricalDataStateLoanBorrow } from '../model/HistoricalDataState'; -import { Log, Warn } from '../utils/Logger'; import { GetGaugeForMarketId } from '../utils/ECGHelper'; import { CreditTransferFile } from '../model/CreditTransfer'; import { HttpGet } from '../utils/HttpHelper'; import { DefiLlamaPriceResponse } from '../model/DefiLlama'; +import logger from '../utils/Logger'; dotenv.config(); let lastCallDefillama = 0; @@ -60,7 +60,7 @@ async function HistoricalDataFetcher() { await LoadConfiguration(); process.title = 'ECG_NODE_HISTORICAL_DATA_FETCHER'; const startDate = Date.now(); - Log('starting'); + logger.debug('starting'); const rpcURL = process.env.RPC_URL; if (!rpcURL) { throw new Error('Cannot find RPC_URL in env'); @@ -75,7 +75,7 @@ async function FetchHistoricalData() { const web3Provider = GetArchiveWeb3Provider(); const currentBlock = await web3Provider.getBlockNumber(); - Log(`fetching data up to block ${currentBlock}`); + logger.debug(`fetching data up to block ${currentBlock}`); const historicalDataDir = path.join(DATA_DIR, 'history'); @@ -120,7 +120,7 @@ async function fetchBlocks(currentBlock: number, historicalDataDir: string, web3 } if (startBlock > currentBlock) { - Log('fetchBlocks: data already up to date'); + logger.debug('fetchBlocks: data already up to date'); return fullHistoricalData.blockTimes; } @@ -128,7 +128,7 @@ async function fetchBlocks(currentBlock: number, historicalDataDir: string, web3 const blockData = await retry(GetBlock, [web3Provider, blockToFetch]); fullHistoricalData.blockTimes[blockToFetch] = blockData.timestamp; - Log(`fetchBlocks: [${blockToFetch}] (${new Date(blockData.timestamp * 1000).toISOString()}) block saved`); + logger.debug(`fetchBlocks: [${blockToFetch}] (${new Date(blockData.timestamp * 1000).toISOString()}) block saved`); WriteJSON(historyFilename, fullHistoricalData); } WriteJSON(historyFilename, fullHistoricalData); @@ -156,7 +156,7 @@ async function fetchCreditTotalSupply( } if (startBlock > currentBlock) { - Log('fetchCreditTotalSupply: data already up to date'); + logger.debug('fetchCreditTotalSupply: data already up to date'); return; } @@ -166,7 +166,7 @@ async function fetchCreditTotalSupply( const totalSupplyAtBlock = await creditTokenContract.targetTotalSupply({ blockTag: blockToFetch }); fullHistoricalData.values[blockToFetch] = norm(totalSupplyAtBlock); fullHistoricalData.blockTimes[blockToFetch] = blockTimes[blockToFetch]; - Log( + logger.debug( `fetchCreditTotalSupply: [${blockToFetch}] (${new Date( blockTimes[blockToFetch] * 1000 ).toISOString()}) total supply : ${fullHistoricalData.values[blockToFetch]}` @@ -197,7 +197,7 @@ async function fetchCreditTotalIssuance( } if (startBlock > currentBlock) { - Log('fetchCreditTotalIssuance: data already up to date'); + logger.debug('fetchCreditTotalIssuance: data already up to date'); return; } @@ -207,7 +207,7 @@ async function fetchCreditTotalIssuance( const totalIssuanceAtBlock = await profitManagerContract.totalIssuance({ blockTag: blockToFetch }); fullHistoricalData.values[blockToFetch] = norm(totalIssuanceAtBlock); fullHistoricalData.blockTimes[blockToFetch] = blockTimes[blockToFetch]; - Log( + logger.debug( `fetchCreditTotalIssuance: [${blockToFetch}] (${new Date( blockTimes[blockToFetch] * 1000 ).toISOString()}) total issuance : ${fullHistoricalData.values[blockToFetch]}` @@ -238,7 +238,7 @@ async function fetchCreditMultiplier( } if (startBlock > currentBlock) { - Log('fetchCreditMultiplier: data already up to date'); + logger.debug('fetchCreditMultiplier: data already up to date'); return; } @@ -248,7 +248,7 @@ async function fetchCreditMultiplier( const creditMultiplier = await retry(() => profitManagerContract.creditMultiplier({ blockTag: blockToFetch }), []); fullHistoricalData.values[blockToFetch] = norm(creditMultiplier); fullHistoricalData.blockTimes[blockToFetch] = blockTimes[blockToFetch]; - Log( + logger.debug( `fetchCreditMultiplier: [${blockToFetch}] (${new Date( blockTimes[blockToFetch] * 1000 ).toISOString()}) credit multiplier: ${fullHistoricalData.values[blockToFetch]}` @@ -281,7 +281,7 @@ async function fetchAverageInterestRate( } if (startBlock > currentBlock) { - Log('fetchAverageInterestRate: data already up to date'); + logger.debug('fetchAverageInterestRate: data already up to date'); return; } @@ -318,7 +318,7 @@ async function fetchAverageInterestRate( fullHistoricalData.values[blockToFetch] = avgInterestRate; fullHistoricalData.blockTimes[blockToFetch] = blockTimes[blockToFetch]; - Log( + logger.debug( `fetchAverageInterestRate: [${blockToFetch}] (${new Date( blockTimes[blockToFetch] * 1000 ).toISOString()}) avg interest rate : ${fullHistoricalData.values[blockToFetch]}` @@ -351,7 +351,7 @@ async function fetchTVL( } if (startBlock > currentBlock) { - Log('fetchTVL: data already up to date'); + logger.debug('fetchTVL: data already up to date'); return; } @@ -400,7 +400,7 @@ async function fetchTVL( let tokenConf = getTokenByAddressNoError(collateralAddress); if (!tokenConf) { tokenConf = await GetERC20Infos(web3Provider, collateralAddress); - Warn( + logger.error( `Token ${collateralAddress} not found in config. ERC20 infos: ${tokenConf.symbol} / ${tokenConf.decimals} decimals` ); } @@ -414,7 +414,7 @@ async function fetchTVL( fullHistoricalData.values[blockToFetch] = tvlInUsd + psmPegTokenValue; fullHistoricalData.blockTimes[blockToFetch] = blockTimes[blockToFetch]; - Log( + logger.debug( `fetchTVL: [${blockToFetch}] (${new Date(blockTimes[blockToFetch] * 1000).toISOString()}) TVL : ${ fullHistoricalData.values[blockToFetch] }. TCL: $${tvlInUsd}, PSM pegTokenValue: $${psmPegTokenValue}` @@ -448,7 +448,7 @@ async function fetchDebtCeilingAndIssuance( } if (startBlock > currentBlock) { - Log('fetchDebtCeilingAndIssuance: data already up to date'); + logger.debug('fetchDebtCeilingAndIssuance: data already up to date'); return; } @@ -482,7 +482,7 @@ async function fetchDebtCeilingAndIssuance( fullHistoricalData.blockTimes[blockToFetch] = blockTimes[blockToFetch]; - Log( + logger.debug( `fetchDebtCeilingAndIssuance: [${blockToFetch}] (${new Date( blockTimes[blockToFetch] * 1000 ).toISOString()}) total debtCeiling: ${totalDebtCeiling} | total issuance: ${totalIssuance}` @@ -515,7 +515,7 @@ async function fetchGaugeWeight( } if (startBlock > currentBlock) { - Log('fetchGaugeWeight: data already up to date'); + logger.debug('fetchGaugeWeight: data already up to date'); return; } @@ -543,7 +543,7 @@ async function fetchGaugeWeight( fullHistoricalData.blockTimes[blockToFetch] = blockTimes[blockToFetch]; - Log( + logger.debug( `fetchGaugeWeight: [${blockToFetch}] (${new Date( blockTimes[blockToFetch] * 1000 ).toISOString()}) total weight: ${totalWeight}` @@ -576,7 +576,7 @@ async function fetchSurplusBuffer( } if (startBlock > currentBlock) { - Log('fetchSurplusBuffer: data already up to date'); + logger.debug('fetchSurplusBuffer: data already up to date'); return; } @@ -605,7 +605,7 @@ async function fetchSurplusBuffer( fullHistoricalData.blockTimes[blockToFetch] = blockTimes[blockToFetch]; - Log( + logger.debug( `fetchSurplusBuffer: [${blockToFetch}] (${new Date( blockTimes[blockToFetch] * 1000 ).toISOString()}) total surplus buffer: ${totalBuffer}` @@ -648,7 +648,7 @@ async function fetchLoansData( } if (startBlock > currentBlock) { - Log('fetchLoansData: data already up to date'); + logger.debug('fetchLoansData: data already up to date'); return; } @@ -747,7 +747,7 @@ async function fetchLoansData( fullHistoricalData.values[blockToFetch].borrowValue = totalBorrowPegToken; fullHistoricalData.values[blockToFetch].totalUnpaidInterests = totalUnpaidInterests; - Log( + logger.debug( `fetchLoansData: [${blockToFetch}] (${new Date( blockTimes[blockToFetch] * 1000 ).toISOString()}) openLoans: ${currentlyOpenedLoans}, borrowValue: ${totalBorrowPegToken}, unpaid interests: ${totalUnpaidInterests}` @@ -784,7 +784,7 @@ async function fetchAPRData( } if (startBlock > currentBlock) { - Log('fetchDebtCeilingAndIssuance: data already up to date'); + logger.debug('fetchDebtCeilingAndIssuance: data already up to date'); return; } @@ -805,7 +805,7 @@ async function fetchAPRData( fullHistoricalData.values[blockToFetch].sharePrice = sharePrice; fullHistoricalData.blockTimes[blockToFetch] = blockTimes[blockToFetch]; - Log( + logger.debug( `fetchAPRData: [${blockToFetch}] (${new Date(blockTimes[blockToFetch] * 1000).toISOString()}) | ` + `rebasingSupply ${fullHistoricalData.values[blockToFetch].rebasingSupply}` + `, totalSupply ${fullHistoricalData.values[blockToFetch].totalSupply}` + @@ -910,7 +910,7 @@ async function GetTokenPriceMultiAtTimestamp( let token = getTokenByAddressNoError(tokenAddress); if (!token) { token = await GetERC20Infos(web3Provider, tokenAddress); - Warn(`Token ${tokenAddress} not found in config. ERC20 infos: ${token.symbol} / ${token.decimals} decimals`); + logger.error(`Token ${tokenAddress} not found in config. ERC20 infos: ${token.symbol} / ${token.decimals} decimals`); } if (token.pendleConfiguration) { @@ -922,7 +922,7 @@ async function GetTokenPriceMultiAtTimestamp( timestamp, web3Provider ); - Log(`GetTokenPriceMulti: price for ${token.symbol} from pendle: ${prices[tokenAddress]}`); + logger.debug(`GetTokenPriceMulti: price for ${token.symbol} from pendle: ${prices[tokenAddress]}`); continue; } @@ -945,17 +945,17 @@ async function GetTokenPriceMultiAtTimestamp( let token = getTokenByAddressNoError(tokenAddress); if (!token) { token = await GetERC20Infos(web3Provider, tokenAddress); - Warn(`Token ${tokenAddress} not found in config. ERC20 infos: ${token.symbol} / ${token.decimals} decimals`); + logger.error(`Token ${tokenAddress} not found in config. ERC20 infos: ${token.symbol} / ${token.decimals} decimals`); } const llamaId = `${llamaNetwork}:${token.mainnetAddress || token.address}`; const llamaPrice = priceResponse.coins[llamaId] ? priceResponse.coins[llamaId].price : 0; prices[tokenAddress] = llamaPrice; - Log(`GetTokenPriceMultiAtTimestamp: price for ${token.symbol} from llama: $${prices[tokenAddress]}`); + logger.debug(`GetTokenPriceMultiAtTimestamp: price for ${token.symbol} from llama: $${prices[tokenAddress]}`); } } - Log(`GetTokenPriceMultiAtTimestamp: ends with ${Object.keys(prices).length} prices`); + logger.debug(`GetTokenPriceMultiAtTimestamp: ends with ${Object.keys(prices).length} prices`); return prices; } @@ -986,7 +986,7 @@ async function GetDefiLlamaPriceAtTimestamp(tokenSymbol: string, tokenId: string return undefined; } - Log(`GetDefiLlamaPriceAtTimestamp: price for ${tokenSymbol} from llama: $${resp.coins[tokenId].price}`); + logger.debug(`GetDefiLlamaPriceAtTimestamp: price for ${tokenSymbol} from llama: $${resp.coins[tokenId].price}`); return resp.coins[tokenId].price; } @@ -1022,7 +1022,7 @@ async function GetPendlePriceAtBlock( } const price = pendlePriceVsAsset * usdPriceBaseAsset; - Log(`GetPendlePriceAtBlock: price for ${tokenSymbol} from pendle: $${price}`); + logger.debug(`GetPendlePriceAtBlock: price for ${tokenSymbol} from pendle: $${price}`); return price; } diff --git a/src/processors/LoanCaller.ts b/src/processors/LoanCaller.ts index a1b2713..eb763f1 100644 --- a/src/processors/LoanCaller.ts +++ b/src/processors/LoanCaller.ts @@ -9,7 +9,7 @@ import { LendingTerm__factory } from '../contracts/types'; import { SendNotifications } from '../utils/Notifications'; import { GetWeb3Provider } from '../utils/Web3Helper'; import { FileMutex } from '../utils/FileMutex'; -import { Log } from '../utils/Logger'; +import logger from '../utils/Logger'; import { LoadConfiguration } from '../config/Config'; const RUN_EVERY_SEC = 15; @@ -21,7 +21,7 @@ async function LoanCaller() { // load external config await LoadConfiguration(); process.title = 'ECG_NODE_LOAN_CALLER'; - Log('starting'); + logger.debug('starting'); const termsFilename = path.join(DATA_DIR, 'terms.json'); const loansFilename = path.join(DATA_DIR, 'loans.json'); checks(termsFilename, loansFilename); @@ -43,7 +43,7 @@ async function LoanCaller() { const loansToCall: { [termAddress: string]: string[] } = {}; const loansToCheck = loanFileData.loans.filter((_) => _.status == LoanStatus.ACTIVE); - Log(`will check ${loansToCheck.length} loans`); + logger.debug(`will check ${loansToCheck.length} loans`); for (const loan of loansToCheck) { const term = termFileData.terms.find((_) => _.termAddress == loan.lendingTermAddress); if (!term) { @@ -55,7 +55,7 @@ async function LoanCaller() { const partialRepayDelayPassed = checkPartialRepayDelayPassed(loan, term); if (termDeprecated || aboveMaxBorrow || partialRepayDelayPassed) { - Log( + logger.debug( `Call needed on Term: ${term.termAddress} / loan ${loan.id} ` + `(termDeprecated: ${termDeprecated}, aboveMaxBorrow: ${aboveMaxBorrow}, partialRepayDelayPassed: ${partialRepayDelayPassed})` ); @@ -69,7 +69,7 @@ async function LoanCaller() { // call if any await callMany(loansToCall); - Log(`sleeping ${RUN_EVERY_SEC} seconds`); + logger.debug(`sleeping ${RUN_EVERY_SEC} seconds`); await sleep(RUN_EVERY_SEC * 1000); } } diff --git a/src/processors/TermOffboarder.ts b/src/processors/TermOffboarder.ts index e9036dc..cf0587e 100644 --- a/src/processors/TermOffboarder.ts +++ b/src/processors/TermOffboarder.ts @@ -17,7 +17,7 @@ import { ethers } from 'ethers'; import { SendNotifications, SendNotificationsList } from '../utils/Notifications'; import { GetERC20Infos, GetWeb3Provider } from '../utils/Web3Helper'; import { FileMutex } from '../utils/FileMutex'; -import { Log, Warn } from '../utils/Logger'; +import logger from '../utils/Logger'; import PriceService from '../services/price/PriceService'; const RUN_EVERY_SEC = 60 * 5; @@ -33,7 +33,7 @@ async function TermOffboarder() { const offboarderConfig = GetNodeConfig().processors.TERM_OFFBOARDER; process.title = 'ECG_NODE_TERM_OFFBOARDER'; - Log('starting'); + logger.debug('starting'); const termsFilename = path.join(DATA_DIR, 'terms.json'); if (!existsSync(termsFilename)) { throw new Error('Cannot start TERM OFFBOARDER without terms file. please sync protocol data'); @@ -55,17 +55,17 @@ async function TermOffboarder() { const checkTermReponse = await checkTermForOffboard(term, offboarderConfig); if (checkTermReponse.termMustBeOffboarded) { if (!offboarderConfig.onlyLogging) { - Log(`[${term.label}]: TERM NEEDS TO BE OFFBOARDED`); + logger.debug(`[${term.label}]: TERM NEEDS TO BE OFFBOARDED`); const web3Provider = GetWeb3Provider(); await offboardProcess(web3Provider, term, offboarderConfig.performCleanup, checkTermReponse.reason); } else { - Log(`[${term.label}]: TERM NEEDS TO BE OFFBOARDED, but 'onlyLogging' is enabled`); + logger.debug(`[${term.label}]: TERM NEEDS TO BE OFFBOARDED, but 'onlyLogging' is enabled`); } } else { - Log(`[${term.label}]: Term is healthy`); + logger.debug(`[${term.label}]: Term is healthy`); } } - Log('Ending'); + logger.debug('Ending'); await WaitUntilScheduled(startDate, RUN_EVERY_SEC); } @@ -84,7 +84,7 @@ async function checkTermForOffboard( permitAllowed: false, protocolToken: false }; - Warn( + logger.error( `Token ${term.collateralAddress} not found in config. ERC20 infos: ${collateralToken.symbol} / ${collateralToken.decimals} decimals` ); } @@ -93,7 +93,9 @@ async function checkTermForOffboard( collateralToken.mainnetAddress || collateralToken.address ); if (!collateralRealPrice) { - Warn(`Cannot find price for ${collateralToken.mainnetAddress || collateralToken.address}. ASSUMING HEALTHY`); + logger.error( + `Cannot find price for ${collateralToken.mainnetAddress || collateralToken.address}. ASSUMING HEALTHY` + ); return { termMustBeOffboarded: false, reason: `Cannot find price for ${collateralToken.mainnetAddress || collateralToken.address}. ASSUMING HEALTHY` @@ -102,18 +104,18 @@ async function checkTermForOffboard( const pegToken = getTokenByAddress(GetPegTokenAddress()); const pegTokenRealPrice = await PriceService.GetTokenPrice(pegToken.mainnetAddress || pegToken.address); if (!pegTokenRealPrice) { - Warn(`Cannot find price for ${pegToken.mainnetAddress || pegToken.address}`); + logger.error(`Cannot find price for ${pegToken.mainnetAddress || pegToken.address}`); return { termMustBeOffboarded: false, reason: `Cannot find price for ${pegToken.mainnetAddress || pegToken.address}` }; } - Log( + logger.debug( `[${term.label}]: ${collateralToken.symbol} price: ${collateralRealPrice} / PegToken price: ${pegTokenRealPrice}` ); const normBorrowRatio = norm(term.maxDebtPerCollateralToken, 36 - collateralToken.decimals); - Log(`[${term.label}]: borrow ratio: ${normBorrowRatio} ${pegToken.symbol} / ${collateralToken.symbol}`); + logger.debug(`[${term.label}]: borrow ratio: ${normBorrowRatio} ${pegToken.symbol} / ${collateralToken.symbol}`); // find the min overcollateralization config for this token let minOvercollateralization = offboarderConfig.defaultMinOvercollateralization; @@ -122,7 +124,7 @@ async function checkTermForOffboard( } const currentOvercollateralization = collateralRealPrice / pegTokenRealPrice / normBorrowRatio; - Log( + logger.debug( `[${term.label}]: current overcollateralization: ${currentOvercollateralization}, min: ${minOvercollateralization}` ); diff --git a/src/processors/TermOnboardingWatcher.ts b/src/processors/TermOnboardingWatcher.ts index 2d68395..598cb61 100644 --- a/src/processors/TermOnboardingWatcher.ts +++ b/src/processors/TermOnboardingWatcher.ts @@ -4,16 +4,10 @@ import { SendNotificationsList } from '../utils/Notifications'; import { norm } from '../utils/TokenUtils'; import OnboardingABI from '../contracts/abi/LendingTermOnboarding.json'; import * as dotenv from 'dotenv'; -import { - GetLendingTermOnboardingAddress, - getTokenByAddress, - getTokenByAddressNoError, - LoadConfiguration, - TokenConfig -} from '../config/Config'; +import { GetLendingTermOnboardingAddress, getTokenByAddressNoError, LoadConfiguration } from '../config/Config'; import { sleep } from '../utils/Utils'; import { GetERC20Infos, GetListenerWeb3Provider } from '../utils/Web3Helper'; -import { Log } from '../utils/Logger'; +import logger from '../utils/Logger'; dotenv.config(); let onboardingContract: Contract | undefined; @@ -22,7 +16,7 @@ const web3Provider = GetListenerWeb3Provider(15000); async function TermOnboardingWatcher() { process.title = 'ECG_NODE_TERM_ONBOARDING_WATCHER'; - Log('starting'); + logger.debug('starting'); const atLeastOneNotificationChannelEnabled = (process.env.WATCHER_TG_BOT_ID != undefined && process.env.WATCHER_TG_CHAT_ID != undefined) || @@ -39,14 +33,14 @@ async function TermOnboardingWatcher() { const onboardingAddress = GetLendingTermOnboardingAddress(); if (onboardingContract) { - Log('resetting contract listener'); + logger.debug('resetting contract listener'); onboardingContract.removeAllListeners(); } onboardingContract = new Contract(onboardingAddress, OnboardingABI, web3Provider); const iface = new Interface(OnboardingABI); - Log(`Create/recreate listener for ProposalCreated events on ${onboardingAddress}`); + logger.debug(`Create/recreate listener for ProposalCreated events on ${onboardingAddress}`); await onboardingContract.on('*', async (event) => { await processEvent(event, web3Provider, iface); }); diff --git a/src/processors/TestnetMarketMaker.ts b/src/processors/TestnetMarketMaker.ts index b03126e..f1b77cd 100644 --- a/src/processors/TestnetMarketMaker.ts +++ b/src/processors/TestnetMarketMaker.ts @@ -9,9 +9,9 @@ import { DATA_DIR } from '../utils/Constants'; import path from 'path'; import fs from 'fs'; import { MarketMakerState } from '../model/MarketMakerState'; -import { Log } from '../utils/Logger'; import { TestnetMarketMakerConfig } from '../model/NodeConfig'; import PriceService from '../services/price/PriceService'; +import logger from '../utils/Logger'; const RUN_EVERY_SEC = 120; @@ -31,7 +31,7 @@ async function TestnetMarketMaker() { // load external config await LoadConfiguration(); process.title = 'ECG_NODE_TESTNET_MARKET_MAKER'; - Log('starting'); + logger.debug('starting'); const config = GetNodeConfig().processors.TESTNET_MARKET_MAKER; const rpcURL = process.env.RPC_URL; @@ -46,7 +46,7 @@ async function TestnetMarketMaker() { } await MakeMarket(config); - Log(`sleeping ${RUN_EVERY_SEC} seconds`); + logger.debug(`sleeping ${RUN_EVERY_SEC} seconds`); await sleep(RUN_EVERY_SEC * 1000); } } @@ -67,12 +67,12 @@ async function MakeMarket(config: TestnetMarketMakerConfig) { const threshold = config.threshold; const pairAddress = config.uniswapPairs[i].poolAddress; const uniswapPair = UniswapV2Pair__factory.connect(pairAddress, web3Provider); - Log(`checking pair ${token0.symbol}-${token1.symbol}`); + logger.debug(`checking pair ${token0.symbol}-${token1.symbol}`); const priceToken0 = await PriceService.GetTokenPrice(token0.mainnetAddress || token0.address); const priceToken1 = await PriceService.GetTokenPrice(token1.mainnetAddress || token1.address); if (!priceToken0 || !priceToken1) { - Log('Cannot market make because real price unknwon', priceToken0, priceToken1); + logger.debug('Cannot market make because real price unknwon', priceToken0, priceToken1); return; } const targetRatio = priceToken1 / priceToken0; @@ -82,10 +82,10 @@ async function MakeMarket(config: TestnetMarketMakerConfig) { Number(reserves[1] * BigInt(10 ** (18 - token1.decimals))); const diff = Math.abs((spotRatio - targetRatio) / targetRatio); - Log(`price diff is ${roundTo(diff * 100, 2)}%. Acceptable diff: ${config.threshold * 100}%`); + logger.debug(`price diff is ${roundTo(diff * 100, 2)}%. Acceptable diff: ${config.threshold * 100}%`); if (diff < threshold) { - Log(`pair almost balanced, no need to swap spotRatio = ${spotRatio} / targetRatio = ${targetRatio}`); + logger.debug(`pair almost balanced, no need to swap spotRatio = ${spotRatio} / targetRatio = ${targetRatio}`); } else { // if we have to swap token0 for token1 if (spotRatio < targetRatio) { @@ -100,7 +100,7 @@ async function MakeMarket(config: TestnetMarketMakerConfig) { Number(reservesAfter[0] * BigInt(10 ** (18 - token0.decimals))) / Number(reservesAfter[1] * BigInt(10 ** (18 - token1.decimals))); } - Log( + logger.debug( `swap ${norm(amountIn, token0.decimals)} ${token0.symbol} -> ${norm(amountOut, token1.decimals)} ${ token1.symbol }` @@ -133,7 +133,7 @@ async function MakeMarket(config: TestnetMarketMakerConfig) { Number(reservesAfter[0] * BigInt(10 ** (18 - token0.decimals))) / Number(reservesAfter[1] * BigInt(10 ** (18 - token1.decimals))); } - Log( + logger.debug( `swap ${norm(amountIn, token1.decimals)} ${token1.symbol} -> ${norm(amountOut, token0.decimals)} ${ token0.symbol }` diff --git a/src/processors/UserSlasher.ts b/src/processors/UserSlasher.ts index adf5821..7609748 100644 --- a/src/processors/UserSlasher.ts +++ b/src/processors/UserSlasher.ts @@ -10,7 +10,7 @@ import { UserSlasherState } from '../model/UserSlasherState'; import { SendNotificationsList } from '../utils/Notifications'; import { GetWeb3Provider } from '../utils/Web3Helper'; import { FileMutex } from '../utils/FileMutex'; -import { Log } from '../utils/Logger'; +import logger from '../utils/Logger'; const RUN_EVERY_SEC = 600; const SLASH_DELAY_MS = 12 * 60 * 60 * 1000; // try slashing same user every 12 hours @@ -25,7 +25,7 @@ async function UserSlasher() { // load external config await LoadConfiguration(); process.title = 'ECG_NODE_USER_SLASHER'; - Log('starting'); + logger.debug('starting'); const config = GetNodeConfig().processors.USER_SLASHER; if (!process.env.RPC_URL) { @@ -59,13 +59,13 @@ async function UserSlasher() { if (user.lastLossApplied < gauge.lastLoss && user.weight > BigInt(config.minSizeToSlash) * 1n ** 18n) { const userLastState = userSlasherState.gauges[gauge.address]?.users[user.address]; if (userLastState && userLastState.lastCheckedTimestamp + SLASH_DELAY_MS > Date.now()) { - Log( + logger.debug( `user ${user.address} for gauge ${gauge.address} was already tried at ${new Date( userLastState.lastCheckedTimestamp ).toISOString()}` ); } else { - Log(`slashing user ${user.address} for gauge ${gauge.address}`); + logger.debug(`slashing user ${user.address} for gauge ${gauge.address}`); try { const web3Provider = GetWeb3Provider(); const signer = new ethers.Wallet(process.env.ETH_PRIVATE_KEY, web3Provider); @@ -98,7 +98,7 @@ async function UserSlasher() { lastCheckedTimestamp: Date.now() }; - Log(`Cannot slash user ${user.address} for gauge ${gauge.address}: ${e.reason}`); + logger.debug(`Cannot slash user ${user.address} for gauge ${gauge.address}: ${e.reason}`); slashMsgfields.push({ fieldName: `${gauge.address} / ${user.address}`, @@ -121,7 +121,7 @@ async function UserSlasher() { } } - // Log(`sending ${calls.length} applyGaugeLoss using Multicall3`); + // logger.debug(`sending ${calls.length} applyGaugeLoss using Multicall3`); // do & wait multicall of guildToken.applyGaugeLoss(gauge, user) // const multicallResponse = await multicallContract.aggregate3(calls, { gasLimit: slashCounter * 200000 }); diff --git a/src/services/cache/CacheService.ts b/src/services/cache/CacheService.ts index 3a0625e..64549f2 100644 --- a/src/services/cache/CacheService.ts +++ b/src/services/cache/CacheService.ts @@ -1,5 +1,3 @@ -import { Log } from '../../utils/Logger'; - const cache: SimpleCache = {}; interface SimpleCache { @@ -32,11 +30,8 @@ export default class SimpleCacheService { static async GetAndCache(key: string, fct: () => Promise, cacheDurationMs: number): Promise { let cached = SimpleCacheService.Get(key); if (!cached) { - // Log(`CACHE MISS FOR ${key}`); cached = await fct(); SimpleCacheService.Set(key, cached, cacheDurationMs); - } else { - // Log(`CACHE HIT FOR ${key}`); } return cached; diff --git a/src/services/price/PriceService.ts b/src/services/price/PriceService.ts index d271d67..2f5ed64 100644 --- a/src/services/price/PriceService.ts +++ b/src/services/price/PriceService.ts @@ -1,7 +1,6 @@ import { median } from 'simple-statistics'; import { DexEnum, TokenConfig, getAllTokens, getTokenByAddressNoError } from '../../config/Config'; import { ECG_NODE_API_URI, GET_PRICES_FROM_API, NETWORK } from '../../utils/Constants'; -import { Log, Warn } from '../../utils/Logger'; import { GetERC20Infos, GetWeb3Provider } from '../../utils/Web3Helper'; import { UniswapV3Pool__factory } from '../../contracts/types'; import SimpleCacheService from '../cache/CacheService'; @@ -15,6 +14,7 @@ import { CoincapAssetsResponse } from '../../model/Coincap'; import { TokenListResponse } from '../../model/OpenOceanApi'; import { DexGuruTokensResponse } from '../../model/DexGuru'; import { SendNotifications } from '../../utils/Notifications'; +import logger from '../../utils/Logger'; interface PriceResult { source: string; @@ -32,7 +32,7 @@ export default class PriceService { ); if (allPrices[tokenAddress] == undefined) { - Warn(`GetTokenPrice: price not found in cache for ${tokenAddress}, fetching price from defillama only`); + logger.error(`GetTokenPrice: price not found in cache for ${tokenAddress}, fetching price from defillama only`); const unkTokenPrice = await SimpleCacheService.GetAndCache( `unk-token-price-${tokenAddress}`, async () => { @@ -62,13 +62,13 @@ async function LoadConfigTokenPrices(): Promise<{ [tokenAddress: string]: number // this is done so that every node of a server will query the local API which will query the prices only once if (GET_PRICES_FROM_API && ECG_NODE_API_URI) { // the process is not the API, will call the API - Log('LoadConfigTokenPrices: loading prices from local API'); + logger.debug('LoadConfigTokenPrices: loading prices from local API'); const nodeApiPriceUrl = `${ECG_NODE_API_URI}/api/protocol/prices`; allPrices = await HttpGet<{ [tokenAddress: string]: number }>(nodeApiPriceUrl); } else { - Log('LoadConfigTokenPrices: loading configuration token prices'); + logger.debug('LoadConfigTokenPrices: loading configuration token prices'); const tokens = getAllTokens(); - Log(`LoadConfigTokenPrices: will fetch price for ${tokens.length} tokens`); + logger.debug(`LoadConfigTokenPrices: will fetch price for ${tokens.length} tokens`); const genericTokenToFetch: TokenConfig[] = []; for (const token of tokens) { @@ -99,7 +99,7 @@ async function LoadConfigTokenPrices(): Promise<{ [tokenAddress: string]: number if (token.pendleConfiguration) { // fetch price using pendle api allPrices[token.address] = await GetPendleApiMarketPrice(token.pendleConfiguration.market); - Log(`LoadConfigTokenPrices: price for ${token.symbol} from pendle: ${allPrices[token.address]}`); + logger.debug(`LoadConfigTokenPrices: price for ${token.symbol} from pendle: ${allPrices[token.address]}`); continue; } @@ -155,13 +155,13 @@ async function LoadConfigTokenPrices(): Promise<{ [tokenAddress: string]: number const msg = `Token ${token.symbol} price from source ${priceSource} seems wrong (${Math.round( absDeviation * 100 )}% off): $${tokenPrice} vs median $${allPrices[token.address]}`; - Warn(msg); + logger.error(msg); await SendNotifications('PriceService', `${token.symbol} ${priceSource} price deviation`, msg); } } } - Log( + logger.debug( `LoadConfigTokenPrices: price for ${token.symbol}: $${allPrices[token.address]}. Medianed from ${ prices.length } sources: ${prices}` @@ -170,7 +170,7 @@ async function LoadConfigTokenPrices(): Promise<{ [tokenAddress: string]: number } } - Log(`LoadConfigTokenPrices: ends with ${Object.keys(allPrices).length} prices`); + logger.debug(`LoadConfigTokenPrices: ends with ${Object.keys(allPrices).length} prices`); return allPrices; } @@ -193,7 +193,7 @@ async function getSafeWethPrice(): Promise { if (price == 0) { throw new Error('getSafeWethPrice: error when fetching weth price'); } - Log(`getSafeWethPrice: returning 1 WETH = $${price} from binance`); + logger.debug(`getSafeWethPrice: returning 1 WETH = $${price} from binance`); return price; }, 5 * 60 * 1000 // 5 min cache @@ -278,7 +278,7 @@ async function GetDexPriceMulti(tokens: TokenConfig[], wethPriceUsd: number): Pr } const tokenPrice = median(pricesForToken); - Log( + logger.debug( `GetDexPriceMulti: price for ${token.symbol} from DEX: $${tokenPrice} from ${pricesForToken.length} prices: ${pricesForToken}` ); prices[token.address] = tokenPrice; @@ -335,14 +335,14 @@ async function GetDefiLlamaPriceMulti(tokens: TokenConfig[]): Promise prices[token.address] = 0; } - Log(`GetCoinCapPriceMulti: price for ${token.symbol} from coincap: $${prices[token.address]}`); + logger.debug(`GetCoinCapPriceMulti: price for ${token.symbol} from coincap: $${prices[token.address]}`); } } } catch (e) { - Warn('Exception calling coincap price api', e); + logger.error('Exception calling coincap price api', e); for (const token of tokens) { if (token.coingeckoId) { prices[token.address] = 0; - Log(`GetCoinCapPriceMulti: price for ${token.symbol} from coincap: $${prices[token.address]}`); + logger.debug(`GetCoinCapPriceMulti: price for ${token.symbol} from coincap: $${prices[token.address]}`); } } } @@ -468,16 +468,16 @@ async function GetOpenOceanPriceMulti(tokens: TokenConfig[]): Promise prices[token.address] = 0; } - Log(`GetDexGuruPriceMulti: price for ${token.symbol} from dex guru: $${prices[token.address]}`); + logger.debug(`GetDexGuruPriceMulti: price for ${token.symbol} from dex guru: $${prices[token.address]}`); } } catch (e) { - Warn('Exception calling DexGuru price api', e); + logger.error('Exception calling DexGuru price api', e); for (const token of tokens) { if (token.coingeckoId) { prices[token.address] = 0; - Log(`GetDexGuruPriceMulti: price for ${token.symbol} from dex guru: $${prices[token.address]}`); + logger.debug(`GetDexGuruPriceMulti: price for ${token.symbol} from dex guru: $${prices[token.address]}`); } } } @@ -577,15 +577,15 @@ async function GetOneInchPriceMulti(tokens: TokenConfig[]): Promise prices[token.address] = 0; } - Log(`GetOneInchPriceMulti: price for ${token.symbol} from 1inch: $${prices[token.address]}`); + logger.debug(`GetOneInchPriceMulti: price for ${token.symbol} from 1inch: $${prices[token.address]}`); } } catch (e) { - Warn('Exception calling 1inch price api', e); + logger.error('Exception calling 1inch price api', e); for (const token of tokens) { if (token.coingeckoId) { prices[token.address] = 0; - Log(`GetOneInchPriceMulti: price for ${token.symbol} from 1inch: $${prices[token.address]}`); + logger.debug(`GetOneInchPriceMulti: price for ${token.symbol} from 1inch: $${prices[token.address]}`); } } } diff --git a/src/spam-watcher/SpamWatcher.ts b/src/spam-watcher/SpamWatcher.ts index fed011f..6ec9dea 100644 --- a/src/spam-watcher/SpamWatcher.ts +++ b/src/spam-watcher/SpamWatcher.ts @@ -5,12 +5,12 @@ import LendingTermAbi from '../contracts/abi/LendingTerm.json'; import PSMAbi from '../contracts/abi/SimplePSM.json'; import { GuildToken__factory, LendingTerm__factory } from '../contracts/types'; import { GetListenerWeb3Provider } from '../utils/Web3Helper'; -import { Log, Warn } from '../utils/Logger'; import { EventData } from '../utils/EventQueue'; import { SendNotificationsSpam } from '../utils/Notifications'; import { MulticallWrapper } from 'ethers-multicall-provider'; import { buildTxUrl } from '../utils/Utils'; import { HttpGet } from '../utils/HttpHelper'; +import logger from '../utils/Logger'; dotenv.config(); const GUILD_TOKEN_ADDRESS = '0xb8ae64F191F829fC00A4E923D460a8F2E0ba3978'; @@ -27,7 +27,7 @@ let psmContracts: Contract[] = []; let termsContracts: Contract[] = []; export function StartSpamEventListener() { const provider = GetListenerWeb3Provider(5000); - Log('Starting/restarting spam listener'); + logger.debug('Starting/restarting spam listener'); StartGuildTokenListener(provider); StartLendingTermListener(provider); StartPSMListener(provider); @@ -40,9 +40,9 @@ export function StartGuildTokenListener(provider: JsonRpcProvider) { guildTokenContract.removeAllListeners(); } - Log('Started the event listener'); + logger.debug('Started the event listener'); guildTokenContract = new Contract(GUILD_TOKEN_ADDRESS, GuildTokenAbi, provider); - Log(`Starting listener on guild token ${GUILD_TOKEN_ADDRESS}`); + logger.debug(`Starting listener on guild token ${GUILD_TOKEN_ADDRESS}`); const iface = new Interface(GuildTokenAbi); @@ -53,7 +53,7 @@ export function StartGuildTokenListener(provider: JsonRpcProvider) { const parsed = iface.parseLog(event.log); if (!parsed) { - Log('Could not parse event', { event }); + logger.debug('Could not parse event', { event }); return; } @@ -85,7 +85,7 @@ export function StartLendingTermListener(provider: JsonRpcProvider) { // get all lives terms const guildTokenContract = GuildToken__factory.connect(GUILD_TOKEN_ADDRESS, provider); guildTokenContract.liveGauges().then((liveTerms) => { - Log(`Liveterms: ${liveTerms.length}`); + logger.debug(`Liveterms: ${liveTerms.length}`); // find all gauges with debt ceiling const multicallProvider = MulticallWrapper.wrap(GetListenerWeb3Provider(5000)); Promise.all( @@ -103,11 +103,11 @@ export function StartLendingTermListener(provider: JsonRpcProvider) { } } - Log(`Starting terms listener for ${termsWithDebtCeiling.length}/${liveTerms.length} terms`); + logger.debug(`Starting terms listener for ${termsWithDebtCeiling.length}/${liveTerms.length} terms`); for (const lendingTermAddress of termsWithDebtCeiling) { const termContract = new Contract(lendingTermAddress, LendingTermAbi, provider); termsContracts.push(termContract); - Log(`Starting listener on term ${lendingTermAddress}`); + logger.debug(`Starting listener on term ${lendingTermAddress}`); const iface = new Interface(LendingTermAbi); termContract.removeAllListeners(); @@ -117,7 +117,7 @@ export function StartLendingTermListener(provider: JsonRpcProvider) { const parsed = iface.parseLog(event.log); if (!parsed) { - Log('Could not parse event', { event }); + logger.debug('Could not parse event', { event }); return; } @@ -150,7 +150,7 @@ export function StartPSMListener(provider: JsonRpcProvider) { for (const psm of PSM_ADDRESSES) { const psmContract = new Contract(psm.address, PSMAbi, provider); psmContracts.push(psmContract); - Log(`Starting listener on psm ${psm.address}`); + logger.debug(`Starting listener on psm ${psm.address}`); const iface = new Interface(PSMAbi); psmContract.removeAllListeners(); @@ -160,7 +160,7 @@ export function StartPSMListener(provider: JsonRpcProvider) { const parsed = iface.parseLog(event.log); if (!parsed) { - Log('Could not parse event', { event }); + logger.debug('Could not parse event', { event }); return; } @@ -205,7 +205,7 @@ async function SendSpamNotif(event: EventData) { const formattedNotif = await formatNotif(event, fields); await SendNotificationsSpam(formattedNotif.title, formattedNotif.text, formattedNotif.fields); } catch (e) { - Warn('Error sending notification to spam', e); + logger.error('Error sending notification to spam', e); } } diff --git a/src/utils/FileMutex.ts b/src/utils/FileMutex.ts index 03bbcad..b4ba869 100644 --- a/src/utils/FileMutex.ts +++ b/src/utils/FileMutex.ts @@ -3,7 +3,7 @@ import path from 'path'; import { DATA_DIR } from './Constants'; import { sleep } from './Utils'; import { SendNotifications } from './Notifications'; -import { Log } from './Logger'; +import logger from './Logger'; const lockFile = path.join(DATA_DIR, 'data.lock'); @@ -21,7 +21,7 @@ export class FileMutex { if (lockedMsg == `LOCKED BY ${process.title}`) { isLocked = false; } else { - Log(`WaitForUnlock: ${lockedMsg}`); + logger.debug(`WaitForUnlock: ${lockedMsg}`); if (waitDuration > 15 * 60 * 1000) { // if more than 10 min wait, notify and throw await SendNotifications('FileMutex', 'Timeout', `WaitForUnlock: timeout waiting on ${lockedMsg}`); diff --git a/src/utils/Logger.ts b/src/utils/Logger.ts index e807473..09769e4 100644 --- a/src/utils/Logger.ts +++ b/src/utils/Logger.ts @@ -2,6 +2,7 @@ import * as dotenv from 'dotenv'; import winston from 'winston'; import LokiTransport from 'winston-loki'; dotenv.config(); +import os from 'os'; const logger = winston.createLogger({ level: 'silly', @@ -26,7 +27,7 @@ if (process.env.LOKI_URI && process.env.LOKI_LOGIN && process.env.LOKI_PWD) { host: process.env.LOKI_URI, format: winston.format.printf((log) => log.message), json: true, - labels: getDefaultMetadata(), + labels: getLokiLabels(), basicAuth: `${process.env.LOKI_LOGIN}:${process.env.LOKI_PWD}`, useWinstonMetaAsLabels: false, batching: true @@ -34,33 +35,12 @@ if (process.env.LOKI_URI && process.env.LOKI_LOGIN && process.env.LOKI_PWD) { ); } -function getDefaultMetadata() { +function getLokiLabels() { return { app: process.env.APP_NAME, - market: process.env.MARKET_ID + market: process.env.MARKET_ID, + host: os.hostname() }; } export default logger; - -///// OLD CODE - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export function Log(msg: string, ...args: any[]) { - const marketId = process.env.MARKET_ID; - if (marketId) { - console.log(`[${process.title}] | MARKET ${marketId} | ${msg}`, ...args); - } else { - console.log(`[${process.title}] | ${msg}`, ...args); - } -} - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export function Warn(msg: string, ...args: any[]) { - const marketId = process.env.MARKET_ID; - if (marketId) { - console.warn(`[${process.title}] | MARKET ${marketId} | ${msg}`, ...args); - } else { - console.warn(`[${process.title}] | ${msg}`, ...args); - } -} diff --git a/src/utils/TelegramHelper.ts b/src/utils/TelegramHelper.ts index 8fecdda..ba2ff68 100644 --- a/src/utils/TelegramHelper.ts +++ b/src/utils/TelegramHelper.ts @@ -1,5 +1,5 @@ import { sleep, truncateString } from './Utils'; -import { Log } from './Logger'; +import logger from './Logger'; import { HttpPost } from './HttpHelper'; import axios from 'axios'; @@ -28,7 +28,7 @@ async function CallTelegram(botId: string, chatId: string, msg: string, isMarkdo }; const timeToWait = 3000 - (Date.now() - lastTGCall); if (timeToWait > 0) { - Log(`SendTelegramMessage: waiting ${timeToWait} ms before calling telegram`); + logger.silly(`SendTelegramMessage: waiting ${timeToWait} ms before calling telegram`); await sleep(timeToWait); } let mustReCall = true; @@ -37,20 +37,20 @@ async function CallTelegram(botId: string, chatId: string, msg: string, isMarkdo try { await HttpPost(url, body, config); - Log('Message sent to telegram with success'); + logger.debug('Message sent to telegram with success'); lastTGCall = Date.now(); } catch (err) { if (axios.isAxiosError(err) && err.response) { - // Log(err.response?.data) + // logger.debug(err.response?.data) if (!err?.response) { - Log('SendTelegramMessage: No Server Response', err); + logger.warn('SendTelegramMessage: No Server Response', err); throw err; } else if (err.response?.status === 429) { - Log('SendTelegramMessage: rate limited, sleeping 5 sec', err); + logger.warn('SendTelegramMessage: rate limited, sleeping 5 sec', err); await sleep(5000); mustReCall = true; } else { - Log('SendTelegramMessage: Unknown error', err); + logger.warn('SendTelegramMessage: Unknown error', err); } } else throw err; } From 014705192af4113bc05f46b0a13784d02800af05 Mon Sep 17 00:00:00 2001 From: Alfu Date: Tue, 4 Jun 2024 09:30:36 +0200 Subject: [PATCH 4/7] sends only >= info to loki --- src/utils/Logger.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/Logger.ts b/src/utils/Logger.ts index 09769e4..fa3eefb 100644 --- a/src/utils/Logger.ts +++ b/src/utils/Logger.ts @@ -23,7 +23,7 @@ const logger = winston.createLogger({ if (process.env.LOKI_URI && process.env.LOKI_LOGIN && process.env.LOKI_PWD) { logger.add( new LokiTransport({ - level: 'debug', + level: 'info', host: process.env.LOKI_URI, format: winston.format.printf((log) => log.message), json: true, From 823a88da240686c2274ce0255a27bc45eb2589d6 Mon Sep 17 00:00:00 2001 From: Alfu Date: Tue, 4 Jun 2024 09:52:58 +0200 Subject: [PATCH 5/7] clean logs --- src/ECGNode.ts | 4 - src/datafetch/ECGDataFetcher.ts | 6 +- src/datafetch/EventProcessor.ts | 5 +- src/datafetch/EventWatcher.ts | 10 +- src/datafetch/fetchers/AuctionsFetcher.ts | 2 +- src/datafetch/fetchers/GaugesFetcher.ts | 2 +- src/datafetch/fetchers/LastActivityFetcher.ts | 2 +- src/datafetch/fetchers/LendingTermsFetcher.ts | 2 +- src/datafetch/fetchers/LoansFetcher.ts | 2 +- .../fetchers/TermsProposalFetcher.ts | 2 +- src/processors/AuctionBidder.ts | 10 +- src/processors/HistoricalDataFetcher.ts | 10 +- src/processors/LoanCaller.ts | 4 +- src/processors/TermOffboarder.ts | 8 +- src/processors/TermOnboardingWatcher.ts | 121 ------------------ src/processors/UserSlasher.ts | 6 +- src/services/price/PriceService.ts | 4 + 17 files changed, 41 insertions(+), 159 deletions(-) delete mode 100644 src/processors/TermOnboardingWatcher.ts diff --git a/src/ECGNode.ts b/src/ECGNode.ts index bed23cf..142c09b 100644 --- a/src/ECGNode.ts +++ b/src/ECGNode.ts @@ -62,10 +62,6 @@ async function startProcessors(nodeConfig: NodeConfig) { startWithSpawn('UserSlasher', 'ECG_NODE_USER_SLASHER'); await sleep(5000); } - // if (nodeConfig.processors.TERM_ONBOARDING_WATCHER.enabled) { - // startWithSpawn('TermOnboardingWatcher'); - // await sleep(5000); - // } if (nodeConfig.processors.TESTNET_MARKET_MAKER.enabled) { startWithSpawn('TestnetMarketMaker', 'ECG_NODE_TESTNET_MARKET_MAKER'); await sleep(5000); diff --git a/src/datafetch/ECGDataFetcher.ts b/src/datafetch/ECGDataFetcher.ts index e0d5020..f5a3044 100644 --- a/src/datafetch/ECGDataFetcher.ts +++ b/src/datafetch/ECGDataFetcher.ts @@ -26,7 +26,7 @@ export async function FetchECGData() { const dtStart = performance.now(); const web3Provider = GetWeb3Provider(); const currentBlock = await web3Provider.getBlockNumber(); - logger.debug(`FetchECGData: fetching data up to block ${currentBlock}`); + logger.info(`FetchECGData: fetching data up to block ${currentBlock}`); const syncData: SyncData = getSyncData(); logger.debug('FetchECGData: start fetching'); @@ -53,9 +53,9 @@ export async function FetchECGData() { logger.debug(`FetchECGData: fetchProposals data took: ${(performance.now() - fetchStart).toFixed(1)} ms`); WriteJSON(path.join(DATA_DIR, 'sync.json'), syncData); const durationMs = performance.now() - dtStart; - logger.debug(`FetchECGData: finished fetching. Fetch duration: ${durationMs.toFixed(1)} ms`); + logger.info(`FetchECGData: finished fetching. Fetch duration: ${durationMs.toFixed(1)} ms`); } catch (e) { - logger.debug('FetchECGData: unknown failure', e); + logger.error('FetchECGData: unknown failure', e); lastFetch = 0; await SendNotifications('Data Fetcher', 'Unknown exception when fetching data', JSON.stringify(e)); } finally { diff --git a/src/datafetch/EventProcessor.ts b/src/datafetch/EventProcessor.ts index ba1ff3a..be6f746 100644 --- a/src/datafetch/EventProcessor.ts +++ b/src/datafetch/EventProcessor.ts @@ -26,6 +26,7 @@ async function ProcessAsync(event: EventData) { logger.debug(`NEW EVENT DETECTED AT BLOCK ${event.block}: ${event.eventName}`); if (mustUpdateProtocol(event)) { if (lastBlockFetched < event.block) { + logger.info(`Important event ${event.eventName} detected at block ${event.block}. Restarting FetchECGData`); await FetchECGData(); lastBlockFetched = event.block; @@ -35,9 +36,7 @@ async function ProcessAsync(event: EventData) { } } - const msg = 'Updated backend data\n' + `Tx: ${buildTxUrl(event.txHash)}`; - - logger.debug(msg); + logger.debug('Updated backend data\n' + `Tx: ${buildTxUrl(event.txHash)}`); } } diff --git a/src/datafetch/EventWatcher.ts b/src/datafetch/EventWatcher.ts index 9ceed2a..1c46d42 100644 --- a/src/datafetch/EventWatcher.ts +++ b/src/datafetch/EventWatcher.ts @@ -22,7 +22,7 @@ let termFactoryContract: Contract | undefined = undefined; let termsContracts: Contract[] = []; export function StartEventListener(onlyTerms = false) { const provider = GetListenerWeb3Provider(5000); - logger.debug(`Starting/restarting events listener, onlyTerms: ${onlyTerms}`); + logger.info(`Starting/restarting events listener, onlyTerms: ${onlyTerms}`); if (onlyTerms) { StartLendingTermListener(provider); } else { @@ -54,7 +54,7 @@ export function StartGuildTokenListener(provider: JsonRpcProvider) { const parsed = iface.parseLog(event.log); if (!parsed) { - logger.debug('Could not parse event', { event }); + logger.error('Could not parse event', { event }); return; } @@ -110,7 +110,7 @@ export function StartOnboardingListener(provider: JsonRpcProvider) { const parsed = onboardingContract?.interface.parseLog(event.log); if (!parsed) { - logger.debug('Could not parse event', { event }); + logger.error('Could not parse event', { event }); return; } @@ -145,7 +145,7 @@ export function StartTermFactoryListener(provider: JsonRpcProvider) { const parsed = termFactoryContract?.interface.parseLog(event.log); if (!parsed) { - logger.debug('Could not parse event', { event }); + logger.error('Could not parse event', { event }); return; } @@ -189,7 +189,7 @@ export function StartLendingTermListener(provider: JsonRpcProvider) { const parsed = iface.parseLog(event.log); if (!parsed) { - logger.debug('Could not parse event', { event }); + logger.error('Could not parse event', { event }); return; } diff --git a/src/datafetch/fetchers/AuctionsFetcher.ts b/src/datafetch/fetchers/AuctionsFetcher.ts index 38d6bd6..bec2fb0 100644 --- a/src/datafetch/fetchers/AuctionsFetcher.ts +++ b/src/datafetch/fetchers/AuctionsFetcher.ts @@ -20,7 +20,7 @@ export default class AuctionsFetcher { syncData: SyncData, currentBlock: number ) { - logger.debug('FetchECGData[Auctions]: starting'); + logger.info('FetchECGData[Auctions]: starting'); let alreadySavedAuctions: Auction[] = []; const auctionsFilePath = path.join(DATA_DIR, 'auctions.json'); if (fs.existsSync(auctionsFilePath)) { diff --git a/src/datafetch/fetchers/GaugesFetcher.ts b/src/datafetch/fetchers/GaugesFetcher.ts index 6e07a11..9c7fe46 100644 --- a/src/datafetch/fetchers/GaugesFetcher.ts +++ b/src/datafetch/fetchers/GaugesFetcher.ts @@ -13,7 +13,7 @@ import logger from '../../utils/Logger'; export default class GaugesFetcher { static async fetchAndSaveGauges(web3Provider: JsonRpcProvider, syncData: SyncData, currentBlock: number) { - logger.debug('FetchECGData[Gauges]: starting'); + logger.info('FetchECGData[Gauges]: starting'); let sinceBlock = GetDeployBlock(); if (syncData.gaugeSync) { sinceBlock = syncData.gaugeSync.lastBlockFetched + 1; diff --git a/src/datafetch/fetchers/LastActivityFetcher.ts b/src/datafetch/fetchers/LastActivityFetcher.ts index 990c125..144660e 100644 --- a/src/datafetch/fetchers/LastActivityFetcher.ts +++ b/src/datafetch/fetchers/LastActivityFetcher.ts @@ -37,7 +37,7 @@ export default class LastActivityFetcher { protocolData: ProtocolData, terms: LendingTerm[] ) { - logger.debug('FetchECGData[LastActivity]: starting'); + logger.info('FetchECGData[LastActivity]: starting'); // read already saved activity let alreadySavedActivities: LastActivity[] = []; diff --git a/src/datafetch/fetchers/LendingTermsFetcher.ts b/src/datafetch/fetchers/LendingTermsFetcher.ts index c57be31..2d5185d 100644 --- a/src/datafetch/fetchers/LendingTermsFetcher.ts +++ b/src/datafetch/fetchers/LendingTermsFetcher.ts @@ -19,7 +19,7 @@ import { SendNotifications } from '../../utils/Notifications'; export default class LendingTermsFetcher { static async fetchAndSaveTerms(web3Provider: JsonRpcProvider, currentBlock: number) { - logger.debug('FetchECGData[Terms]: starting'); + logger.info('FetchECGData[Terms]: starting'); const multicallProvider = MulticallWrapper.wrap(web3Provider); const guildTokenContract = GuildToken__factory.connect(GetGuildTokenAddress(), multicallProvider); const gauges = await GetGaugeForMarketId(guildTokenContract, MARKET_ID, false); diff --git a/src/datafetch/fetchers/LoansFetcher.ts b/src/datafetch/fetchers/LoansFetcher.ts index 07eeefb..ffe38c7 100644 --- a/src/datafetch/fetchers/LoansFetcher.ts +++ b/src/datafetch/fetchers/LoansFetcher.ts @@ -20,7 +20,7 @@ export default class LoansFetcher { syncData: SyncData, currentBlock: number ) { - logger.debug('FetchECGData[Loans]: starting'); + logger.info('FetchECGData[Loans]: starting'); let alreadySavedLoans: Loan[] = []; const loansFilePath = path.join(DATA_DIR, 'loans.json'); diff --git a/src/datafetch/fetchers/TermsProposalFetcher.ts b/src/datafetch/fetchers/TermsProposalFetcher.ts index da04467..26ab69a 100644 --- a/src/datafetch/fetchers/TermsProposalFetcher.ts +++ b/src/datafetch/fetchers/TermsProposalFetcher.ts @@ -26,7 +26,7 @@ const nodeConfig = GetNodeConfig(); export default class TermsProposalFetcher { static async fetchProposals(web3Provider: JsonRpcProvider, syncData: SyncData, currentBlock: number) { - logger.debug('FetchECGData[Proposals]: starting'); + logger.info('FetchECGData[Proposals]: starting'); let allProposals: Proposal[] = []; const proposalsFilePath = path.join(DATA_DIR, 'proposals.json'); diff --git a/src/processors/AuctionBidder.ts b/src/processors/AuctionBidder.ts index 76287c0..088b76b 100644 --- a/src/processors/AuctionBidder.ts +++ b/src/processors/AuctionBidder.ts @@ -36,7 +36,7 @@ async function AuctionBidder() { // load external config await LoadConfiguration(); process.title = 'ECG_NODE_AUCTION_BIDDER'; - logger.debug('starting'); + logger.info('starting'); const auctionBidderConfig = GetNodeConfig().processors.AUCTION_BIDDER; const auctionsFilename = path.join(DATA_DIR, 'auctions.json'); @@ -66,11 +66,11 @@ async function AuctionBidder() { const bidDetail = await getBidDetails(auction.auctionHouseAddress, web3Provider, auction.loanId); if (bidDetail.auctionEnded) { - logger.debug('Auction ended, will not try to bid'); + logger.info('Auction ended, will not try to bid'); continue; } if (bidDetail.creditAsked == 0n && auctionBidderConfig.enableForgive) { - logger.debug(`AuctionBidder[${auction.loanId}]: will forgive auction.`); + logger.info(`AuctionBidder[${auction.loanId}]: will forgive auction.`); await processForgive(auction, web3Provider); continue; } @@ -84,7 +84,7 @@ async function AuctionBidder() { ); if (estimatedProfit >= auctionBidderConfig.minProfitPegToken) { - logger.debug(`AuctionBidder[${auction.loanId}]: will bid on auction for estimated profit: ${estimatedProfit}`); + logger.info(`AuctionBidder[${auction.loanId}]: will bid on auction for estimated profit: ${estimatedProfit}`); await processBid( auction, term, @@ -118,7 +118,7 @@ async function getBidDetails( auctionEnded: false }; } catch (e) { - logger.debug('getBidDetail exception:', e); + logger.warn('getBidDetail exception:', e); return { collateralReceived: -1n, creditAsked: -1n, diff --git a/src/processors/HistoricalDataFetcher.ts b/src/processors/HistoricalDataFetcher.ts index 0e9367f..9fe5c45 100644 --- a/src/processors/HistoricalDataFetcher.ts +++ b/src/processors/HistoricalDataFetcher.ts @@ -60,7 +60,7 @@ async function HistoricalDataFetcher() { await LoadConfiguration(); process.title = 'ECG_NODE_HISTORICAL_DATA_FETCHER'; const startDate = Date.now(); - logger.debug('starting'); + logger.info('starting'); const rpcURL = process.env.RPC_URL; if (!rpcURL) { throw new Error('Cannot find RPC_URL in env'); @@ -910,7 +910,9 @@ async function GetTokenPriceMultiAtTimestamp( let token = getTokenByAddressNoError(tokenAddress); if (!token) { token = await GetERC20Infos(web3Provider, tokenAddress); - logger.error(`Token ${tokenAddress} not found in config. ERC20 infos: ${token.symbol} / ${token.decimals} decimals`); + logger.error( + `Token ${tokenAddress} not found in config. ERC20 infos: ${token.symbol} / ${token.decimals} decimals` + ); } if (token.pendleConfiguration) { @@ -945,7 +947,9 @@ async function GetTokenPriceMultiAtTimestamp( let token = getTokenByAddressNoError(tokenAddress); if (!token) { token = await GetERC20Infos(web3Provider, tokenAddress); - logger.error(`Token ${tokenAddress} not found in config. ERC20 infos: ${token.symbol} / ${token.decimals} decimals`); + logger.error( + `Token ${tokenAddress} not found in config. ERC20 infos: ${token.symbol} / ${token.decimals} decimals` + ); } const llamaId = `${llamaNetwork}:${token.mainnetAddress || token.address}`; const llamaPrice = priceResponse.coins[llamaId] ? priceResponse.coins[llamaId].price : 0; diff --git a/src/processors/LoanCaller.ts b/src/processors/LoanCaller.ts index eb763f1..869ff96 100644 --- a/src/processors/LoanCaller.ts +++ b/src/processors/LoanCaller.ts @@ -21,7 +21,7 @@ async function LoanCaller() { // load external config await LoadConfiguration(); process.title = 'ECG_NODE_LOAN_CALLER'; - logger.debug('starting'); + logger.info('starting'); const termsFilename = path.join(DATA_DIR, 'terms.json'); const loansFilename = path.join(DATA_DIR, 'loans.json'); checks(termsFilename, loansFilename); @@ -55,7 +55,7 @@ async function LoanCaller() { const partialRepayDelayPassed = checkPartialRepayDelayPassed(loan, term); if (termDeprecated || aboveMaxBorrow || partialRepayDelayPassed) { - logger.debug( + logger.info( `Call needed on Term: ${term.termAddress} / loan ${loan.id} ` + `(termDeprecated: ${termDeprecated}, aboveMaxBorrow: ${aboveMaxBorrow}, partialRepayDelayPassed: ${partialRepayDelayPassed})` ); diff --git a/src/processors/TermOffboarder.ts b/src/processors/TermOffboarder.ts index cf0587e..bd3e27f 100644 --- a/src/processors/TermOffboarder.ts +++ b/src/processors/TermOffboarder.ts @@ -15,7 +15,7 @@ import { TermOffboarderConfig } from '../model/NodeConfig'; import { LendingTermOffboarding__factory } from '../contracts/types'; import { ethers } from 'ethers'; import { SendNotifications, SendNotificationsList } from '../utils/Notifications'; -import { GetERC20Infos, GetWeb3Provider } from '../utils/Web3Helper'; +import { GetWeb3Provider } from '../utils/Web3Helper'; import { FileMutex } from '../utils/FileMutex'; import logger from '../utils/Logger'; import PriceService from '../services/price/PriceService'; @@ -33,7 +33,7 @@ async function TermOffboarder() { const offboarderConfig = GetNodeConfig().processors.TERM_OFFBOARDER; process.title = 'ECG_NODE_TERM_OFFBOARDER'; - logger.debug('starting'); + logger.info('starting'); const termsFilename = path.join(DATA_DIR, 'terms.json'); if (!existsSync(termsFilename)) { throw new Error('Cannot start TERM OFFBOARDER without terms file. please sync protocol data'); @@ -55,11 +55,11 @@ async function TermOffboarder() { const checkTermReponse = await checkTermForOffboard(term, offboarderConfig); if (checkTermReponse.termMustBeOffboarded) { if (!offboarderConfig.onlyLogging) { - logger.debug(`[${term.label}]: TERM NEEDS TO BE OFFBOARDED`); + logger.info(`[${term.label}]: TERM NEEDS TO BE OFFBOARDED`); const web3Provider = GetWeb3Provider(); await offboardProcess(web3Provider, term, offboarderConfig.performCleanup, checkTermReponse.reason); } else { - logger.debug(`[${term.label}]: TERM NEEDS TO BE OFFBOARDED, but 'onlyLogging' is enabled`); + logger.info(`[${term.label}]: TERM NEEDS TO BE OFFBOARDED, but 'onlyLogging' is enabled`); } } else { logger.debug(`[${term.label}]: Term is healthy`); diff --git a/src/processors/TermOnboardingWatcher.ts b/src/processors/TermOnboardingWatcher.ts deleted file mode 100644 index 598cb61..0000000 --- a/src/processors/TermOnboardingWatcher.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { Contract, Interface, JsonRpcProvider } from 'ethers'; -import { LendingTerm__factory } from '../contracts/types'; -import { SendNotificationsList } from '../utils/Notifications'; -import { norm } from '../utils/TokenUtils'; -import OnboardingABI from '../contracts/abi/LendingTermOnboarding.json'; -import * as dotenv from 'dotenv'; -import { GetLendingTermOnboardingAddress, getTokenByAddressNoError, LoadConfiguration } from '../config/Config'; -import { sleep } from '../utils/Utils'; -import { GetERC20Infos, GetListenerWeb3Provider } from '../utils/Web3Helper'; -import logger from '../utils/Logger'; -dotenv.config(); - -let onboardingContract: Contract | undefined; - -const web3Provider = GetListenerWeb3Provider(15000); - -async function TermOnboardingWatcher() { - process.title = 'ECG_NODE_TERM_ONBOARDING_WATCHER'; - logger.debug('starting'); - - const atLeastOneNotificationChannelEnabled = - (process.env.WATCHER_TG_BOT_ID != undefined && process.env.WATCHER_TG_CHAT_ID != undefined) || - process.env.WATCHER_DISCORD_WEBHOOK_URL != undefined; - - if (!atLeastOneNotificationChannelEnabled) { - throw new Error('At least one notification channel must be enabled'); - } - - // eslint-disable-next-line no-constant-condition - while (true) { - // load external config - await LoadConfiguration(); - const onboardingAddress = GetLendingTermOnboardingAddress(); - - if (onboardingContract) { - logger.debug('resetting contract listener'); - onboardingContract.removeAllListeners(); - } - - onboardingContract = new Contract(onboardingAddress, OnboardingABI, web3Provider); - const iface = new Interface(OnboardingABI); - - logger.debug(`Create/recreate listener for ProposalCreated events on ${onboardingAddress}`); - await onboardingContract.on('*', async (event) => { - await processEvent(event, web3Provider, iface); - }); - - await sleep(10 * 60 * 1000); - } -} - -async function processEvent(event: any, web3Provider: JsonRpcProvider, iface: Interface) { - const parsed = iface.parseLog(event.log); - if (parsed == null) { - return; - } - - if (parsed.name != 'ProposalCreated') { - return; - } - - /*description = string.concat( - "[", - Strings.toString(block.number), - "]", - " Enable term ", - Strings.toHexString(term) - );*/ - // extract term address from description - const termAddress = parsed.args.description.split(' Enable term ')[1]; - const proposalId = parsed.args.proposalId; - const proposer = parsed.args.proposer; - const lendingTerm = LendingTerm__factory.connect(termAddress, web3Provider); - - const params = await lendingTerm.getParameters(); - - let collateralTokenStr = params.collateralToken; - - let foundToken = getTokenByAddressNoError(collateralTokenStr); - if (!foundToken) { - foundToken = await GetERC20Infos(web3Provider, collateralTokenStr); - } - - if (foundToken) { - collateralTokenStr += ` (${foundToken.symbol})`; - } - - await SendNotificationsList( - 'TermOnboarderWatcher', - `New term ${termAddress} is proposed`, - [ - { - fieldName: 'Proposal Id', - fieldValue: proposalId.toString(10) - }, - { - fieldName: 'Proposer', - fieldValue: proposer - }, - { - fieldName: 'Collateral', - fieldValue: collateralTokenStr - }, - { - fieldName: 'Hard Cap', - fieldValue: params.hardCap.toString(10) - }, - { - fieldName: 'Interest rate', - fieldValue: norm(params.interestRate).toString() - }, - { - fieldName: 'maxDebtPerCollateralToken', - fieldValue: params.maxDebtPerCollateralToken.toString(10) - } - ], - true - ); -} - -TermOnboardingWatcher(); diff --git a/src/processors/UserSlasher.ts b/src/processors/UserSlasher.ts index 7609748..786efd8 100644 --- a/src/processors/UserSlasher.ts +++ b/src/processors/UserSlasher.ts @@ -25,7 +25,7 @@ async function UserSlasher() { // load external config await LoadConfiguration(); process.title = 'ECG_NODE_USER_SLASHER'; - logger.debug('starting'); + logger.info('starting'); const config = GetNodeConfig().processors.USER_SLASHER; if (!process.env.RPC_URL) { @@ -65,7 +65,7 @@ async function UserSlasher() { ).toISOString()}` ); } else { - logger.debug(`slashing user ${user.address} for gauge ${gauge.address}`); + logger.info(`slashing user ${user.address} for gauge ${gauge.address}`); try { const web3Provider = GetWeb3Provider(); const signer = new ethers.Wallet(process.env.ETH_PRIVATE_KEY, web3Provider); @@ -98,7 +98,7 @@ async function UserSlasher() { lastCheckedTimestamp: Date.now() }; - logger.debug(`Cannot slash user ${user.address} for gauge ${gauge.address}: ${e.reason}`); + logger.warn(`Cannot slash user ${user.address} for gauge ${gauge.address}: ${e.reason}`); slashMsgfields.push({ fieldName: `${gauge.address} / ${user.address}`, diff --git a/src/services/price/PriceService.ts b/src/services/price/PriceService.ts index 2f5ed64..1b3f7ff 100644 --- a/src/services/price/PriceService.ts +++ b/src/services/price/PriceService.ts @@ -168,6 +168,10 @@ async function LoadConfigTokenPrices(): Promise<{ [tokenAddress: string]: number ); } } + + for (const token of tokens) { + logger.info(`${token.symbol} PRICE: ${allPrices[token.address]}`); + } } logger.debug(`LoadConfigTokenPrices: ends with ${Object.keys(allPrices).length} prices`); From 1ad32e6a43a6d731eaf2bac77026f375803fd4e0 Mon Sep 17 00:00:00 2001 From: Alfu Date: Tue, 4 Jun 2024 09:58:48 +0200 Subject: [PATCH 6/7] lending terms fetcher now send multicall by 500 calls --- src/datafetch/fetchers/LendingTermsFetcher.ts | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/src/datafetch/fetchers/LendingTermsFetcher.ts b/src/datafetch/fetchers/LendingTermsFetcher.ts index 2d5185d..3b6c155 100644 --- a/src/datafetch/fetchers/LendingTermsFetcher.ts +++ b/src/datafetch/fetchers/LendingTermsFetcher.ts @@ -25,7 +25,8 @@ export default class LendingTermsFetcher { const gauges = await GetGaugeForMarketId(guildTokenContract, MARKET_ID, false); const profitManagerContract = ProfitManager__factory.connect(GetProfitManagerAddress(), web3Provider); // eslint-disable-next-line @typescript-eslint/no-explicit-any - const promises: any[] = []; + let promises: any[] = []; + const results: any[] = []; promises.push(profitManagerContract.minBorrow()); promises.push(guildTokenContract.totalTypeWeight(MARKET_ID)); for (const lendingTermAddress of gauges) { @@ -37,25 +38,34 @@ export default class LendingTermsFetcher { promises.push(lendingTermContract.auctionHouse()); promises.push(guildTokenContract.getGaugeWeight(lendingTermAddress)); promises.push(profitManagerContract.termSurplusBuffer(lendingTermAddress)); + + if (promises.length >= 500) { + logger.debug(`FetchECGData[Terms]: sending ${promises.length} multicall`); + const subResults = await Promise.all(promises); + logger.debug('FetchECGData[Terms]: end multicall'); + results.push(...subResults); + promises = []; + } } // wait the promises logger.debug(`FetchECGData[Terms]: sending ${promises.length} multicall`); - await Promise.all(promises); + const subResults = await Promise.all(promises); + results.push(...subResults); logger.debug('FetchECGData[Terms]: end multicall'); const lendingTerms: LendingTerm[] = []; let cursor = 0; - const minBorrow: bigint = await promises[cursor++]; - const totalTypeWeight: bigint = await promises[cursor++]; + const minBorrow: bigint = results[cursor++]; + const totalTypeWeight: bigint = results[cursor++]; for (const lendingTermAddress of gauges) { - // read promises in the same order as the multicall - const termParameters: LendingTermType.LendingTermParamsStructOutput = await promises[cursor++]; - const issuance: bigint = await promises[cursor++]; - const debtCeiling: bigint = await promises[cursor++]; - const auctionHouseAddress: string = await promises[cursor++]; - const gaugeWeight: bigint = await promises[cursor++]; - const termSurplusBuffer: bigint = await promises[cursor++]; + // read results in the same order as the multicall + const termParameters: LendingTermType.LendingTermParamsStructOutput = results[cursor++]; + const issuance: bigint = results[cursor++]; + const debtCeiling: bigint = results[cursor++]; + const auctionHouseAddress: string = results[cursor++]; + const gaugeWeight: bigint = results[cursor++]; + const termSurplusBuffer: bigint = results[cursor++]; const realCap = termParameters.hardCap > debtCeiling ? debtCeiling : termParameters.hardCap; const availableDebt = issuance > realCap ? 0n : realCap - issuance; From 8b577dfa96f5ace18a57c2824e6b18f3948a950e Mon Sep 17 00:00:00 2001 From: Alfu Date: Mon, 10 Jun 2024 09:30:32 +0200 Subject: [PATCH 7/7] Revert "lending terms fetcher now send multicall by 500 calls" This reverts commit 1ad32e6a43a6d731eaf2bac77026f375803fd4e0. --- src/datafetch/fetchers/LendingTermsFetcher.ts | 32 +++++++------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/src/datafetch/fetchers/LendingTermsFetcher.ts b/src/datafetch/fetchers/LendingTermsFetcher.ts index 3b6c155..2d5185d 100644 --- a/src/datafetch/fetchers/LendingTermsFetcher.ts +++ b/src/datafetch/fetchers/LendingTermsFetcher.ts @@ -25,8 +25,7 @@ export default class LendingTermsFetcher { const gauges = await GetGaugeForMarketId(guildTokenContract, MARKET_ID, false); const profitManagerContract = ProfitManager__factory.connect(GetProfitManagerAddress(), web3Provider); // eslint-disable-next-line @typescript-eslint/no-explicit-any - let promises: any[] = []; - const results: any[] = []; + const promises: any[] = []; promises.push(profitManagerContract.minBorrow()); promises.push(guildTokenContract.totalTypeWeight(MARKET_ID)); for (const lendingTermAddress of gauges) { @@ -38,34 +37,25 @@ export default class LendingTermsFetcher { promises.push(lendingTermContract.auctionHouse()); promises.push(guildTokenContract.getGaugeWeight(lendingTermAddress)); promises.push(profitManagerContract.termSurplusBuffer(lendingTermAddress)); - - if (promises.length >= 500) { - logger.debug(`FetchECGData[Terms]: sending ${promises.length} multicall`); - const subResults = await Promise.all(promises); - logger.debug('FetchECGData[Terms]: end multicall'); - results.push(...subResults); - promises = []; - } } // wait the promises logger.debug(`FetchECGData[Terms]: sending ${promises.length} multicall`); - const subResults = await Promise.all(promises); - results.push(...subResults); + await Promise.all(promises); logger.debug('FetchECGData[Terms]: end multicall'); const lendingTerms: LendingTerm[] = []; let cursor = 0; - const minBorrow: bigint = results[cursor++]; - const totalTypeWeight: bigint = results[cursor++]; + const minBorrow: bigint = await promises[cursor++]; + const totalTypeWeight: bigint = await promises[cursor++]; for (const lendingTermAddress of gauges) { - // read results in the same order as the multicall - const termParameters: LendingTermType.LendingTermParamsStructOutput = results[cursor++]; - const issuance: bigint = results[cursor++]; - const debtCeiling: bigint = results[cursor++]; - const auctionHouseAddress: string = results[cursor++]; - const gaugeWeight: bigint = results[cursor++]; - const termSurplusBuffer: bigint = results[cursor++]; + // read promises in the same order as the multicall + const termParameters: LendingTermType.LendingTermParamsStructOutput = await promises[cursor++]; + const issuance: bigint = await promises[cursor++]; + const debtCeiling: bigint = await promises[cursor++]; + const auctionHouseAddress: string = await promises[cursor++]; + const gaugeWeight: bigint = await promises[cursor++]; + const termSurplusBuffer: bigint = await promises[cursor++]; const realCap = termParameters.hardCap > debtCeiling ? debtCeiling : termParameters.hardCap; const availableDebt = issuance > realCap ? 0n : realCap - issuance;