diff --git a/package-lock.json b/package-lock.json index 054e36428..962c9ebc4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -86,6 +86,7 @@ "jsdom": "^27.3.0", "knip": "^5.75.1", "prettier": "^3.7.4", + "tsx": "^4.21.0", "typescript": "^5.9.3", "typescript-eslint": "^8.50.0", "vite": "^7.3.0", @@ -206,7 +207,6 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.5.tgz", "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", @@ -713,7 +713,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=18" }, @@ -760,7 +759,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=18" } @@ -1556,7 +1554,6 @@ "resolved": "https://registry.npmjs.org/@hey-api/openapi-ts/-/openapi-ts-0.89.0.tgz", "integrity": "sha512-6cnHppAR6vM8osyWqiCoHy35J3CqFz114ggOLHwaTb795XUnzoP/pdbvyz+TBpukY08QQh69kHMAXdi2Kuq9Ow==", "license": "MIT", - "peer": true, "dependencies": { "@hey-api/codegen-core": "^0.4.0", "@hey-api/json-schema-ref-parser": "1.2.2", @@ -1655,7 +1652,6 @@ "resolved": "https://registry.npmjs.org/@hyperjump/browser/-/browser-1.3.1.tgz", "integrity": "sha512-Le5XZUjnVqVjkgLYv6yyWgALat/0HpB1XaCPuCZ+GCFki9NvXloSZITIJ0H+wRW7mb9At1SxvohKBbNQbrr/cw==", "license": "MIT", - "peer": true, "dependencies": { "@hyperjump/json-pointer": "^1.1.0", "@hyperjump/uri": "^1.2.0", @@ -4025,7 +4021,6 @@ "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.90.12.tgz", "integrity": "sha512-graRZspg7EoEaw0a8faiUASCyJrqjKPdqJ9EwuDRUF9mEYJ1YPczI9H+/agJ0mOJkPCJDk0lsz5QTrLZ/jQ2rg==", "license": "MIT", - "peer": true, "dependencies": { "@tanstack/query-core": "5.90.12" }, @@ -4059,7 +4054,6 @@ "resolved": "https://registry.npmjs.org/@tanstack/react-router/-/react-router-1.141.6.tgz", "integrity": "sha512-qWFxi2D6eGc1L03RzUuhyEOplZ7Q6q62YOl7Of9Y0q4YjwQwxRm4zxwDVtvUIoy4RLVCpqp5UoE+Nxv2PY9trg==", "license": "MIT", - "peer": true, "dependencies": { "@tanstack/history": "1.141.0", "@tanstack/react-store": "^0.8.0", @@ -4130,7 +4124,6 @@ "resolved": "https://registry.npmjs.org/@tanstack/router-core/-/router-core-1.141.6.tgz", "integrity": "sha512-AqH61axLq2xFaM+B0veGQ4OOzMzr2Ih+qXzBmGRy5e0wMJkr1efPZXLF0K7nEjF++bmL/excew2Br6v9xrZ/5g==", "license": "MIT", - "peer": true, "dependencies": { "@tanstack/history": "1.141.0", "@tanstack/store": "^0.8.0", @@ -4221,7 +4214,6 @@ "integrity": "sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.10.4", "@babel/runtime": "^7.12.5", @@ -4457,7 +4449,6 @@ "integrity": "sha512-W609buLVRVmeW693xKfzHeIV6nJGGz98uCPfeXI1ELMLXVeKYZ9m15fAMSaUPBHYLGFsVRcMmSCksQOrZV9BYA==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -4467,7 +4458,6 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.7.tgz", "integrity": "sha512-MWtvHrGZLFttgeEj28VXHxpmwYbor/ATPYbBfSFZEIRK0ecCFLl2Qo55z52Hss+UV9CRN7trSeq1zbgx7YDWWg==", "license": "MIT", - "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -4478,7 +4468,6 @@ "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", "devOptional": true, "license": "MIT", - "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -4527,7 +4516,6 @@ "integrity": "sha512-6/cmF2piao+f6wSxUsJLZjck7OQsYyRtcOZS02k7XINSNlz93v6emM8WutDQSXnroG2xwYlEVHJI+cPA7CPM3Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.50.0", "@typescript-eslint/types": "8.50.0", @@ -4933,7 +4921,6 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -5309,7 +5296,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001718", "electron-to-chromium": "^1.5.160", @@ -5703,8 +5689,7 @@ "version": "3.2.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/d3-color": { "version": "3.1.0", @@ -5763,7 +5748,6 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", - "peer": true, "engines": { "node": ">=12" } @@ -6037,8 +6021,7 @@ "version": "4.2.1", "resolved": "https://registry.npmjs.org/dexie/-/dexie-4.2.1.tgz", "integrity": "sha512-Ckej0NS6jxQ4Po3OrSQBFddayRhTCic2DoCAG5zacOfOVB9P2Q5Xc5uL/nVa7ZVs+HdMnvUPzLFCB/JwpB6Csg==", - "license": "Apache-2.0", - "peer": true + "license": "Apache-2.0" }, "node_modules/dexie-react-hooks": { "version": "4.2.0", @@ -6087,7 +6070,8 @@ "version": "3.1.7", "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.1.7.tgz", "integrity": "sha512-VaTstWtsneJY8xzy7DekmYWEOZcmzIe3Qb3zPd4STve1OBTa+e+WmS1ITQec1fZYXI3HCsOZZiSMpG6oxoWMWQ==", - "license": "(MPL-2.0 OR Apache-2.0)" + "license": "(MPL-2.0 OR Apache-2.0)", + "peer": true }, "node_modules/dotenv": { "version": "17.2.3", @@ -6418,7 +6402,6 @@ "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.2.tgz", "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -6549,7 +6532,6 @@ "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", "dev": true, "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } @@ -8227,7 +8209,6 @@ "integrity": "sha512-GtldT42B8+jefDUC4yUKAvsaOrH7PDHmZxZXNgF2xMmymjUbRYJvpAybZAKEmXDGTM0mCsz8duOa4vTm5AY2Kg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@acemir/cssom": "^0.9.28", "@asamuzakjp/dom-selector": "^6.7.6", @@ -8784,7 +8765,6 @@ "integrity": "sha512-L0WhttDl+2BOsybvEOLK7fW3UA0OQ0IQ2d6Zl2x/a6vVRs3bAY0ECOSHHeL5jD+SbOpOCUEi0y1DgHEn9Qn1AQ==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/parser": "^7.25.4", "@babel/types": "^7.25.4", @@ -8820,6 +8800,7 @@ "resolved": "https://registry.npmjs.org/marked/-/marked-14.0.0.tgz", "integrity": "sha512-uIj4+faQ+MgHgwUW1l2PsPglZLOLOT1uErt06dAPtx2kjteLAkbsd/0FiYg/MGS+i7ZKLb7w2WClxHkzOOuryQ==", "license": "MIT", + "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -9518,7 +9499,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -9545,7 +9525,6 @@ "resolved": "https://registry.npmjs.org/preact/-/preact-10.26.9.tgz", "integrity": "sha512-SSjF9vcnF27mJK1XyFMNJzFd5u3pQiATFqoaDy03XuN00u4ziveVVEGt5RKJrDR8MHE/wJo9Nnad56RLzS2RMA==", "license": "MIT", - "peer": true, "funding": { "type": "opencollective", "url": "https://opencollective.com/preact" @@ -9689,7 +9668,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.3.tgz", "integrity": "sha512-Ku/hhYbVjOQnXDZFv2+RibmLFGwFdeeKHFcOTlrt7xplBnya5OGn/hIRDsqDiSUcfORsDC7MPxwork8jBwsIWA==", "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -9699,7 +9677,6 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.3.tgz", "integrity": "sha512-yELu4WmLPw5Mr/lmeEpox5rw3RETacE++JgHqQzd2dg+YbJuat3jH4ingc+WPZhxaoFzdv9y33G+F7Nl5O0GBg==", "license": "MIT", - "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -10027,7 +10004,6 @@ "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.44.0.tgz", "integrity": "sha512-qHcdEzLCiktQIfwBq420pn2dP+30uzqYxv9ETm91wdt2R9AFcWfjNAmje4NWlnCIQ5RMTzVf0ZyisOKqHR6RwA==", "license": "MIT", - "peer": true, "dependencies": { "@types/estree": "1.0.8" }, @@ -10201,7 +10177,6 @@ "resolved": "https://registry.npmjs.org/seroval/-/seroval-1.4.0.tgz", "integrity": "sha512-BdrNXdzlofomLTiRnwJTSEAaGKyHHZkbMXIywOh7zlzp4uZnXErEwl9XZ+N1hJSNpeTtNxWvVwN0wUzAIQ4Hpg==", "license": "MIT", - "peer": true, "engines": { "node": ">=10" } @@ -10417,6 +10392,7 @@ "resolved": "https://registry.npmjs.org/solid-js/-/solid-js-1.9.10.tgz", "integrity": "sha512-Coz956cos/EPDlhs6+jsdTxKuJDPT7B5SVIWgABwROyxjY7Xbr8wkzD68Et+NxnV7DLJ3nJdAC2r9InuV/4Jew==", "license": "MIT", + "peer": true, "dependencies": { "csstype": "^3.1.0", "seroval": "~1.3.0", @@ -10438,6 +10414,7 @@ "resolved": "https://registry.npmjs.org/seroval-plugins/-/seroval-plugins-1.3.3.tgz", "integrity": "sha512-16OL3NnUBw8JG1jBLUoZJsLnQq0n5Ua6aHalhJK4fMQkz1lqR7Osz1sA30trBtd9VUDc2NgkuRCn8+/pBwqZ+w==", "license": "MIT", + "peer": true, "engines": { "node": ">=10" }, @@ -10795,8 +10772,7 @@ "version": "4.1.18", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.1.18.tgz", "integrity": "sha512-4+Z+0yiYyEtUVCScyfHCxOYP06L5Ne+JiHhY2IjR2KWMIWhJOYZKLSGZaP5HkZ8+bY0cxfzwDE5uOmzFXyIwxw==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/tailwindcss-animate": { "version": "1.0.7", @@ -11006,12 +10982,12 @@ "license": "0BSD" }, "node_modules/tsx": { - "version": "4.20.3", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.3.tgz", - "integrity": "sha512-qjbnuR9Tr+FJOMBqJCW5ehvIo/buZq7vH7qD7JziU98h6l3qGy0a/yPFjwO+y0/T7GFpNgNAvEcPPVfyT8rrPQ==", + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", + "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", "license": "MIT", "dependencies": { - "esbuild": "~0.25.0", + "esbuild": "~0.27.0", "get-tsconfig": "^4.7.5" }, "bin": { @@ -11024,6 +11000,447 @@ "fsevents": "~2.3.3" } }, + "node_modules/tsx/node_modules/@esbuild/aix-ppc64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.1.tgz", + "integrity": "sha512-HHB50pdsBX6k47S4u5g/CaLjqS3qwaOVE5ILsq64jyzgMhLuCuZ8rGzM9yhsAjfjkbgUPMzZEPa7DAp7yz6vuA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-arm": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.1.tgz", + "integrity": "sha512-kFqa6/UcaTbGm/NncN9kzVOODjhZW8e+FRdSeypWe6j33gzclHtwlANs26JrupOntlcWmB0u8+8HZo8s7thHvg==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-arm64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.1.tgz", + "integrity": "sha512-45fuKmAJpxnQWixOGCrS+ro4Uvb4Re9+UTieUY2f8AEc+t7d4AaZ6eUJ3Hva7dtrxAAWHtlEFsXFMAgNnGU9uQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-x64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.1.tgz", + "integrity": "sha512-LBEpOz0BsgMEeHgenf5aqmn/lLNTFXVfoWMUox8CtWWYK9X4jmQzWjoGoNb8lmAYml/tQ/Ysvm8q7szu7BoxRQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/darwin-arm64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.1.tgz", + "integrity": "sha512-veg7fL8eMSCVKL7IW4pxb54QERtedFDfY/ASrumK/SbFsXnRazxY4YykN/THYqFnFwJ0aVjiUrVG2PwcdAEqQQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/darwin-x64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.1.tgz", + "integrity": "sha512-+3ELd+nTzhfWb07Vol7EZ+5PTbJ/u74nC6iv4/lwIU99Ip5uuY6QoIf0Hn4m2HoV0qcnRivN3KSqc+FyCHjoVQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.1.tgz", + "integrity": "sha512-/8Rfgns4XD9XOSXlzUDepG8PX+AVWHliYlUkFI3K3GB6tqbdjYqdhcb4BKRd7C0BhZSoaCxhv8kTcBrcZWP+xg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/freebsd-x64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.1.tgz", + "integrity": "sha512-GITpD8dK9C+r+5yRT/UKVT36h/DQLOHdwGVwwoHidlnA168oD3uxA878XloXebK4Ul3gDBBIvEdL7go9gCUFzQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-arm": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.1.tgz", + "integrity": "sha512-ieMID0JRZY/ZeCrsFQ3Y3NlHNCqIhTprJfDgSB3/lv5jJZ8FX3hqPyXWhe+gvS5ARMBJ242PM+VNz/ctNj//eA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-arm64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.1.tgz", + "integrity": "sha512-W9//kCrh/6in9rWIBdKaMtuTTzNj6jSeG/haWBADqLLa9P8O5YSRDzgD5y9QBok4AYlzS6ARHifAb75V6G670Q==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-ia32": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.1.tgz", + "integrity": "sha512-VIUV4z8GD8rtSVMfAj1aXFahsi/+tcoXXNYmXgzISL+KB381vbSTNdeZHHHIYqFyXcoEhu9n5cT+05tRv13rlw==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-loong64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.1.tgz", + "integrity": "sha512-l4rfiiJRN7sTNI//ff65zJ9z8U+k6zcCg0LALU5iEWzY+a1mVZ8iWC1k5EsNKThZ7XCQ6YWtsZ8EWYm7r1UEsg==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-mips64el": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.1.tgz", + "integrity": "sha512-U0bEuAOLvO/DWFdygTHWY8C067FXz+UbzKgxYhXC0fDieFa0kDIra1FAhsAARRJbvEyso8aAqvPdNxzWuStBnA==", + "cpu": [ + "mips64el" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-ppc64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.1.tgz", + "integrity": "sha512-NzdQ/Xwu6vPSf/GkdmRNsOfIeSGnh7muundsWItmBsVpMoNPVpM61qNzAVY3pZ1glzzAxLR40UyYM23eaDDbYQ==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-riscv64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.1.tgz", + "integrity": "sha512-7zlw8p3IApcsN7mFw0O1Z1PyEk6PlKMu18roImfl3iQHTnr/yAfYv6s4hXPidbDoI2Q0pW+5xeoM4eTCC0UdrQ==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-s390x": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.1.tgz", + "integrity": "sha512-cGj5wli+G+nkVQdZo3+7FDKC25Uh4ZVwOAK6A06Hsvgr8WqBBuOy/1s+PUEd/6Je+vjfm6stX0kmib5b/O2Ykw==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-x64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.1.tgz", + "integrity": "sha512-z3H/HYI9MM0HTv3hQZ81f+AKb+yEoCRlUby1F80vbQ5XdzEMyY/9iNlAmhqiBKw4MJXwfgsh7ERGEOhrM1niMA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.1.tgz", + "integrity": "sha512-wzC24DxAvk8Em01YmVXyjl96Mr+ecTPyOuADAvjGg+fyBpGmxmcr2E5ttf7Im8D0sXZihpxzO1isus8MdjMCXQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/netbsd-x64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.1.tgz", + "integrity": "sha512-1YQ8ybGi2yIXswu6eNzJsrYIGFpnlzEWRl6iR5gMgmsrR0FcNoV1m9k9sc3PuP5rUBLshOZylc9nqSgymI+TYg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.1.tgz", + "integrity": "sha512-5Z+DzLCrq5wmU7RDaMDe2DVXMRm2tTDvX2KU14JJVBN2CT/qov7XVix85QoJqHltpvAOZUAc3ndU56HSMWrv8g==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openbsd-x64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.1.tgz", + "integrity": "sha512-Q73ENzIdPF5jap4wqLtsfh8YbYSZ8Q0wnxplOlZUOyZy7B4ZKW8DXGWgTCZmF8VWD7Tciwv5F4NsRf6vYlZtqg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/sunos-x64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.1.tgz", + "integrity": "sha512-IPUW+y4VIjuDVn+OMzHc5FV4GubIwPnsz6ubkvN8cuhEqH81NovB53IUlrlBkPMEPxvNnf79MGBoz8rZ2iW8HA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-arm64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.1.tgz", + "integrity": "sha512-RIVRWiljWA6CdVu8zkWcRmGP7iRRIIwvhDKem8UMBjPql2TXM5PkDVvvrzMtj1V+WFPB4K7zkIGM7VzRtFkjdg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-ia32": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.1.tgz", + "integrity": "sha512-2BR5M8CPbptC1AK5JbJT1fWrHLvejwZidKx3UMSF0ecHMa+smhi16drIrCEggkgviBwLYd5nwrFLSl5Kho96RQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-x64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.1.tgz", + "integrity": "sha512-d5X6RMYv6taIymSk8JBP+nxv8DQAMY6A51GPgusqLdK9wBz5wWIXy1KjTck6HnjE9hqJzJRdk+1p/t5soSbCtw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/esbuild": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.1.tgz", + "integrity": "sha512-yY35KZckJJuVVPXpvjgxiCuVEJT67F6zDeVTv4rizyPrfGBUpZQsvmxnN+C371c2esD/hNMjj4tpBhuueLN7aA==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.1", + "@esbuild/android-arm": "0.27.1", + "@esbuild/android-arm64": "0.27.1", + "@esbuild/android-x64": "0.27.1", + "@esbuild/darwin-arm64": "0.27.1", + "@esbuild/darwin-x64": "0.27.1", + "@esbuild/freebsd-arm64": "0.27.1", + "@esbuild/freebsd-x64": "0.27.1", + "@esbuild/linux-arm": "0.27.1", + "@esbuild/linux-arm64": "0.27.1", + "@esbuild/linux-ia32": "0.27.1", + "@esbuild/linux-loong64": "0.27.1", + "@esbuild/linux-mips64el": "0.27.1", + "@esbuild/linux-ppc64": "0.27.1", + "@esbuild/linux-riscv64": "0.27.1", + "@esbuild/linux-s390x": "0.27.1", + "@esbuild/linux-x64": "0.27.1", + "@esbuild/netbsd-arm64": "0.27.1", + "@esbuild/netbsd-x64": "0.27.1", + "@esbuild/openbsd-arm64": "0.27.1", + "@esbuild/openbsd-x64": "0.27.1", + "@esbuild/openharmony-arm64": "0.27.1", + "@esbuild/sunos-x64": "0.27.1", + "@esbuild/win32-arm64": "0.27.1", + "@esbuild/win32-ia32": "0.27.1", + "@esbuild/win32-x64": "0.27.1" + } + }, "node_modules/tsx/node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -11133,7 +11550,6 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -11324,7 +11740,6 @@ "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.0.tgz", "integrity": "sha512-dZwN5L1VlUBewiP6H9s2+B3e3Jg96D0vzN+Ry73sOefebhYr9f94wwkMNN/9ouoU8pV1BqA1d1zGk8928cx0rg==", "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", @@ -11878,7 +12293,6 @@ "integrity": "sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/chai": "^5.2.2", "@vitest/expect": "3.2.4", diff --git a/package.json b/package.json index 02b7889fd..64c61583b 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,8 @@ "validate": "npm run format && npm run lint:fix && npm run typecheck && npm run knip", "validate:test": "npm run validate && npm run test", "generate-api": "openapi-ts", - "knip": "knip" + "knip": "knip", + "hydrate-component-library": "tsx scripts/hydrate-component-library.ts public/component_library.original.yaml public/component_library.yaml" }, "dependencies": { "@dnd-kit/core": "^6.3.1", @@ -131,6 +132,7 @@ "typescript": "^5.9.3", "typescript-eslint": "^8.50.0", "vite": "^7.3.0", - "vitest": "^3.0.5" + "vitest": "^3.0.5", + "tsx": "^4.21.0" } } diff --git a/public/component_library.original.yaml b/public/component_library.original.yaml new file mode 100644 index 000000000..9410083cc --- /dev/null +++ b/public/component_library.original.yaml @@ -0,0 +1,183 @@ +annotations: {} +folders: + - name: "Quick start" + components: + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/f328b3f4842838b6578b701d975b22f7908aca48/components/datasets/Chicago_Taxi_Trips/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/57f780b15922061e59833541b71f3d099e710177/components/datasets/Chicago_Taxi_Trips/quick_start_version/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/XGBoost/Train/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/XGBoost/Predict/component.yaml" + + - name: "Basics" + components: + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/c36d9a4893c3caae0d166d9e783af28fcd3cdfe4/components/basics/Format_date_time/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/37d98d43ad3193cf3516c134899f272d9643117c/components/basics/Calculate_hash/component.yaml" + folders: + - name: File system + components: + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/605cefc151797369752b51530a91cc02e7f913f8/components/filesystem/create_directory/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/filesystem/get_file/component.yaml" + + - name: "Datasets" + components: + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/f328b3f4842838b6578b701d975b22f7908aca48/components/datasets/Chicago_Taxi_Trips/component.yaml" + # Has modified default column set. TODO: Perhaps replace with a graph component. + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/57f780b15922061e59833541b71f3d099e710177/components/datasets/Chicago_Taxi_Trips/quick_start_version/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/datasets/HuggingFace/Load_dataset/component.yaml" + + - name: "Data manipulation" + folders: + - name: Parquet + components: + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Select_columns/in_ApacheParquet_format/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Fill_all_missing_values/in_ApacheParquet_format/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Binarize_column/in_ApacheParquet_format/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Transform_DataFrame/in_ApacheParquet_format/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/dataset_manipulation/split_data_into_folds/in_CSV/component.yaml" + - name: CSV + components: + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Select_columns/in_CSV_format/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Binarize_column/in_CSV_format/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Transform_DataFrame/in_CSV_format/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/dataset_manipulation/split_data_into_folds/in_CSV/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml" + - name: JSON lines + components: + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Transform_DataFrame/in_JsonLines_format/component.yaml" + - name: JSON + components: + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/dcf4fdde4876e8d76aa0131ad4d67c47b2b5591a/components/json/Get_element_by_index/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/dcf4fdde4876e8d76aa0131ad4d67c47b2b5591a/components/json/Get_element_by_key/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/json/Query/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/json/Build_dict/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/json/Build_list/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/aecac18d4023c73c561d7f21192253e9593b9932/components/json/Build_list_of_strings/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/bb9d7518b3a23e945c8cc1663942063c6b92c20f/components/json/Build_list_of_integers/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/bb9d7518b3a23e945c8cc1663942063c6b92c20f/components/json/Build_list_of_floats/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/json/Combine_lists/component.yaml" + + - name: "Upload/Download" + components: + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/f328b3f4842838b6578b701d975b22f7908aca48/components/web/Download/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/google-cloud/storage/download/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/6210648f30b2b3a8c01cc10be338da98300efb6b/components/google-cloud/storage/upload_to_unique_uri/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/6210648f30b2b3a8c01cc10be338da98300efb6b/components/google-cloud/storage/upload_to_explicit_uri/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/cca2d8569d01b527df10c629258be04d52eacc43/components/Download_and_upload/IPFS/Download/component.yaml" + + - name: ML frameworks + folders: + - name: "Scikit learn" + components: + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml" + + - name: "XGBoost" + components: + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/XGBoost/Train/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/79d79efc5619692b28bcf14d37718077330ad617/components/XGBoost/Predict/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/XGBoost/Train/from_ApacheParquet/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/79d79efc5619692b28bcf14d37718077330ad617/components/XGBoost/Predict/from_ApacheParquet/component.yaml" + + - name: "PyTorch" + components: + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/PyTorch/Create_fully_connected_network/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/7799eb6ae4c4bb817f7461e2e3ffc83cbe2d50a7/components/PyTorch/Predict/on_CSV/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d2b5b635198ce661c9c58e3a1817c4c018e7c40b/components/PyTorch/Train_PyTorch_model/from_ApacheParquet/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/7799eb6ae4c4bb817f7461e2e3ffc83cbe2d50a7/components/PyTorch/Predict/on_ApacheParquet/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Convert_to_OnnxModel_from_PyTorchScriptModule/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml" + + - name: "Tensorflow" + components: + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/tensorflow/Create_fully_connected_network/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/tensorflow/Train_model_using_Keras/on_ApacheParquet/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/tensorflow/Predict/on_CSV/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/tensorflow/Predict/on_ApacheParquet/component.yaml" + + - name: "CatBoost" + components: + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/CatBoost/Train_regression/from_CSV/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/CatBoost/Train_classifier/from_CSV/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/CatBoost/Predict_values/from_CSV/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/CatBoost/Predict_classes/from_CSV/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/CatBoost/Predict_class_probabilities/from_CSV/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/CatBoost/convert_CatBoostModel_to_ONNX/component.yaml" + + - name: "Vowpal_Wabbit" + components: + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/ML_frameworks/Vowpal_Wabbit/Create_JSON_dataset/from_CSV/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/a2a629e776d5fa0204ce71370cab23282d3e4278/components/ML_frameworks/Vowpal_Wabbit/Train_regression_model/from_VowpalWabbitJsonDataset/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/a2a629e776d5fa0204ce71370cab23282d3e4278/components/ML_frameworks/Vowpal_Wabbit/Predict/from_VowpalWabbitJsonDataset/component.yaml" + + # # Hidden: Need re-testing. Also, TFX is not popular. It's best to properly wrap individual libraries (TFDV, TFMA, etc). + # - name: TFX + # components: + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/deprecated/tfx/ExampleGen/CsvExampleGen/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/deprecated/tfx/StatisticsGen/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/deprecated/tfx/SchemaGen/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/deprecated/tfx/ExampleValidator/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/deprecated/tfx/Transform/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/deprecated/tfx/Trainer/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/deprecated/tfx/Evaluator/component.yaml" + - name: "ML Metrics" + components: + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/9b65c073ae0a131151454c7f9030cb1a553ca7ba/components/ml_metrics/Calculate_regression_metrics/from_CSV/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/89d9d83d77cdbbe521994f5d95893661e14f9172/components/ml_metrics/Calculate_regression_metrics/from_ApacheParquet/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/ml_metrics/Aggregate_regression_metrics/component.yaml" + + - name: "Converters" + components: + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/_converters/ApacheParquet/from_CSV/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/_converters/ApacheParquet/from_TSV/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/_converters/ApacheParquet/from_ApacheArrowFeather/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/_converters/ApacheParquet/to_CSV/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/_converters/ApacheParquet/to_TSV/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/_converters/ApacheParquet/to_ApacheArrowFeather/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/4fca0aa607c00e60d5eb342630acc175e6f51fc2/components/_converters/XGBoostJsonModel/from_XGBoostModel/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/4fca0aa607c00e60d5eb342630acc175e6f51fc2/components/_converters/XGBoostJsonModel/to_XGBoostModel/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/4619f84eebc54230153ba48b0e67ac8446dc31c6/components/_converters/OnnxModel/from_XGBoostJsonModel/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/ae9d04e833d973d785809e7734021b06bcfea9bc/components/_converters/OnnxModel/from_XGBoostModel/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/_converters/OnnxModel/from_TensorflowSavedModel/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/2fc275072568cd0cdf73743ab49aa90928303f2c/components/_converters/OnnxModel/to_TensorflowSavedModel/component.yaml" + - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/f7f79decd10955b8408541f6f695a614538a6901/components/_converters/ScikitLearnPickleModel/to_OnnxModel/component.yaml" + + # # Hidden: Due to auth, these components only work out of the box on GCP (GKE or Vertex AI). But most of our deployments (local, HuggingFace) do not have that. + # - name: "Google Cloud" + # folders: + # - name: Vertex AI + # folders: + # - name: AutoML + # components: + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/47f3621344c884666a926c8a15d77562f1cc5e0a/components/google-cloud/Vertex_AI/AutoML/Tables/Create_dataset/from_CSV/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/00d020c29a144cee7fd35f2d05053addb942f536/components/google-cloud/Vertex_AI/AutoML/Tables/Create_dataset/from_GCS/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/00d020c29a144cee7fd35f2d05053addb942f536/components/google-cloud/Vertex_AI/AutoML/Tables/Create_dataset/from_BigQuery/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/ab85ecc9c30d4d68a2993ca87861f5e531a4f41b/components/google-cloud/Vertex_AI/AutoML/Tables/Train_model/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/a31b7c9652646f2cd035a0b3a23e0723c632521b/components/google-cloud/Vertex_AI/AutoML/Tables/Get_model_tuning_trials/component.yaml" + # - name: Models + # components: + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/2c24c0c0730c818b89f676c4dc5c9d6cb90ab01d/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/25dc317e649a19a53139a08ccbe496a248693fe4/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/719783ef44c04348ea23e247a93021d91cfe602d/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d1e7a3ccf8f8e0324e15922d6fd90d667fc5281b/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/b2cdd60fe93d609111729ef64e79a8b8a2713435/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d45e011ad8b62b4fe36c12289a624e5e1573c68d/components/google-cloud/Vertex_AI/Models/Export/to_GCS/component.yaml" + # - name: Storage + # components: + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/google-cloud/storage/download/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/6210648f30b2b3a8c01cc10be338da98300efb6b/components/google-cloud/storage/upload_to_explicit_uri/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/6210648f30b2b3a8c01cc10be338da98300efb6b/components/google-cloud/storage/upload_to_unique_uri/component.yaml" + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/google-cloud/storage/list/component.yaml" + # - name: AI Platform (legacy) + # folders: + # - name: Optimizer + # components: + # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/1b87c0bdfde5d7ec039401af8561783432731402/components/google-cloud/Optimizer/Suggest_parameter_sets_based_on_measurements/component.yaml" + # # # !!! Google Cloud AI Platform AutoML API cannot be called from Google Cloud Vertex AI Pipelines (~by design) + # # - name: AutoML + # # components: + # # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/3862f752bb825bb8cdcae842f0b294794028376b/components/google-cloud/AutoML/Tables/Create_dataset/from_CSV/component.yaml" + # # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/46b2a4ef4ac530404958839ae397b9fd533cf154/components/google-cloud/AutoML/Tables/Create_dataset/from_GCS/component.yaml" + # # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/247a0e0cf55c28b978c7f8e44fcedba356c12e93/components/google-cloud/AutoML/Tables/Create_dataset/from_BigQuery/component.yaml" + # # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/gcp/automl/create_model_for_tables/component.yaml" diff --git a/public/component_library.yaml b/public/component_library.yaml index 9410083cc..45473f91c 100644 --- a/public/component_library.yaml +++ b/public/component_library.yaml @@ -1,183 +1,429 @@ annotations: {} folders: - - name: "Quick start" + - name: Quick start components: - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/f328b3f4842838b6578b701d975b22f7908aca48/components/datasets/Chicago_Taxi_Trips/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/57f780b15922061e59833541b71f3d099e710177/components/datasets/Chicago_Taxi_Trips/quick_start_version/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/XGBoost/Train/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/XGBoost/Predict/component.yaml" - - - name: "Basics" + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/57f780b15922061e59833541b71f3d099e710177/components/datasets/Chicago_Taxi_Trips/quick_start_version/component.yaml + name: Chicago Taxi Trips dataset + author: Alexey Volkov + digest: 42030acab16b71bbc3ad018563b5aedb94d373e72f7cb2b322bc27dbbee75e1b + fetched_at: '2025-12-15T23:26:53.238Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/XGBoost/Train/component.yaml + name: Train XGBoost model on CSV + author: Alexey Volkov + digest: 5b8bec6716337d7bd8ecafd6dd46bc44527783bc05950cfe03a60206b43c7654 + fetched_at: '2025-12-15T23:26:53.255Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/XGBoost/Predict/component.yaml + name: Xgboost predict on CSV + author: Alexey Volkov + digest: 6fd7196d2061e6f49ec98459c90f4b1e4e63bca21170c70b23f7679921ce01d7 + fetched_at: '2025-12-15T23:26:53.266Z' + - name: Basics components: - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/c36d9a4893c3caae0d166d9e783af28fcd3cdfe4/components/basics/Format_date_time/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/37d98d43ad3193cf3516c134899f272d9643117c/components/basics/Calculate_hash/component.yaml" + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c36d9a4893c3caae0d166d9e783af28fcd3cdfe4/components/basics/Format_date_time/component.yaml + name: Format date and time + author: Alexey Volkov + digest: 7784069b3bd1301674cfb4bbf7170b7db91ffd289138eb2d8838e3368158b70c + fetched_at: '2025-12-15T23:26:53.223Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/37d98d43ad3193cf3516c134899f272d9643117c/components/basics/Calculate_hash/component.yaml + name: Calculate data hash + author: Alexey Volkov + digest: 6afc1b9d9c845fcdf0e9820aa97c9544c0f8b1ec2b7c1cf481975231711f6503 + fetched_at: '2025-12-15T23:26:53.231Z' folders: - name: File system components: - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/605cefc151797369752b51530a91cc02e7f913f8/components/filesystem/create_directory/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/filesystem/get_file/component.yaml" - - - name: "Datasets" + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/605cefc151797369752b51530a91cc02e7f913f8/components/filesystem/create_directory/component.yaml + name: Create directory from files (5) + author: Alexey Volkov + digest: 6fa449465ecf656c02d07622acb5df045c3863acc15c607e1817f4e9f8078134 + fetched_at: '2025-12-15T23:26:53.386Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/filesystem/get_file/component.yaml + name: Get file + author: Alexey Volkov + digest: 4415166532fa69baa33378631802fab630f51ea1677ce6408cef946672f4a9d2 + fetched_at: '2025-12-15T23:26:53.389Z' + - name: Datasets components: - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/f328b3f4842838b6578b701d975b22f7908aca48/components/datasets/Chicago_Taxi_Trips/component.yaml" - # Has modified default column set. TODO: Perhaps replace with a graph component. - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/57f780b15922061e59833541b71f3d099e710177/components/datasets/Chicago_Taxi_Trips/quick_start_version/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/datasets/HuggingFace/Load_dataset/component.yaml" - - - name: "Data manipulation" + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/57f780b15922061e59833541b71f3d099e710177/components/datasets/Chicago_Taxi_Trips/quick_start_version/component.yaml + name: Chicago Taxi Trips dataset + author: Alexey Volkov + digest: 42030acab16b71bbc3ad018563b5aedb94d373e72f7cb2b322bc27dbbee75e1b + fetched_at: '2025-12-15T23:26:53.238Z' + - name: Data manipulation folders: - name: Parquet components: - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Select_columns/in_ApacheParquet_format/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Fill_all_missing_values/in_ApacheParquet_format/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Binarize_column/in_ApacheParquet_format/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Transform_DataFrame/in_ApacheParquet_format/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/dataset_manipulation/split_data_into_folds/in_CSV/component.yaml" + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Select_columns/in_ApacheParquet_format/component.yaml + name: Select columns using Pandas on ApacheParquet data + author: Alexey Volkov + digest: 7cf49b426385086fbb58b2d7c900637439f9dfff0df5661591cb6840b125d57e + fetched_at: '2025-12-15T23:26:53.195Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Fill_all_missing_values/in_ApacheParquet_format/component.yaml + name: Fill all missing values using Pandas on ApacheParquet data + author: Alexey Volkov + digest: 0fd11307bd634351c521d4ae5d32f547e0a77304b17e2eab046e092debb3b2b4 + fetched_at: '2025-12-15T23:26:53.233Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Binarize_column/in_ApacheParquet_format/component.yaml + name: Binarize column using Pandas on ApacheParquet data + author: Alexey Volkov + digest: 3a8d435090075b9064e62974a07baf316063b248b4c175d9e1f42bdf1bdd3439 + fetched_at: '2025-12-15T23:26:53.243Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Transform_DataFrame/in_ApacheParquet_format/component.yaml + name: Pandas Transform DataFrame in ApacheParquet format + author: Alexey Volkov + digest: 45cad5c51cbebcc10c6c552709ba367e3f516f98a33dfaadd776737d69375885 + fetched_at: '2025-12-15T23:26:53.240Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/dataset_manipulation/split_data_into_folds/in_CSV/component.yaml + name: Split table into folds + author: Alexey Volkov + digest: ca3834a0b29b0647f0e48f9d45419bb7ae05c6f4cd6fc98f4d99a28979915284 + fetched_at: '2025-12-15T23:26:53.248Z' - name: CSV components: - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Select_columns/in_CSV_format/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Binarize_column/in_CSV_format/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Transform_DataFrame/in_CSV_format/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/dataset_manipulation/split_data_into_folds/in_CSV/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml" + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Select_columns/in_CSV_format/component.yaml + name: Select columns using Pandas on CSV data + author: Alexey Volkov + digest: 7160345cf18c12fa2a5230c1ed5ceef9f2cc40b8b72d39995c72f99755622bec + fetched_at: '2025-12-15T23:26:53.231Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml + name: Fill all missing values using Pandas on CSV data + author: Alexey Volkov + digest: c873de1f3a95bb7fb17efbbc5a72c4a13a1ef581dc162054707441ddce2c06d2 + fetched_at: '2025-12-15T23:26:53.221Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Binarize_column/in_CSV_format/component.yaml + name: Binarize column using Pandas on CSV data + author: Alexey Volkov + digest: 98e0babb5e2ac915e5c8969b76f61d8e7b78637aef66f7fb508a8f3f35d63cf1 + fetched_at: '2025-12-15T23:26:53.255Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Transform_DataFrame/in_CSV_format/component.yaml + name: Pandas Transform DataFrame in CSV format + author: Alexey Volkov + digest: 6cf3a8e4fedec9f5e92ade9616b06c0717cc7f7622f2da8c6c646d2bed17fa00 + fetched_at: '2025-12-15T23:26:53.211Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/dataset_manipulation/split_data_into_folds/in_CSV/component.yaml + name: Split table into folds + author: Alexey Volkov + digest: ca3834a0b29b0647f0e48f9d45419bb7ae05c6f4cd6fc98f4d99a28979915284 + fetched_at: '2025-12-15T23:26:53.223Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml + name: Split rows into subsets + author: Alexey Volkov + digest: 1d1b505ec8a538347e8c0db84a92f76ebae55c5ef85ff3e91bc1d7eb7a86dc06 + fetched_at: '2025-12-15T23:26:53.205Z' - name: JSON lines components: - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Transform_DataFrame/in_JsonLines_format/component.yaml" + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/pandas/Transform_DataFrame/in_JsonLines_format/component.yaml + name: Transform using Pandas DataFrame on JsonLines data + author: Alexey Volkov + digest: a16b7f91eaf2eecf351c2775d4d33c960457257a90025ac058313946a3c7fc44 + fetched_at: '2025-12-15T23:26:53.232Z' - name: JSON components: - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/dcf4fdde4876e8d76aa0131ad4d67c47b2b5591a/components/json/Get_element_by_index/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/dcf4fdde4876e8d76aa0131ad4d67c47b2b5591a/components/json/Get_element_by_key/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/json/Query/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/json/Build_dict/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/json/Build_list/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/aecac18d4023c73c561d7f21192253e9593b9932/components/json/Build_list_of_strings/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/bb9d7518b3a23e945c8cc1663942063c6b92c20f/components/json/Build_list_of_integers/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/bb9d7518b3a23e945c8cc1663942063c6b92c20f/components/json/Build_list_of_floats/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/json/Combine_lists/component.yaml" - - - name: "Upload/Download" + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/dcf4fdde4876e8d76aa0131ad4d67c47b2b5591a/components/json/Get_element_by_index/component.yaml + name: Get element by index from JSON + author: Alexey Volkov + digest: c7b09bd3bf9cf9e42e22bedcc4822c32c31f81f92e45c019f9cca626961cf2a2 + fetched_at: '2025-12-15T23:26:53.233Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/dcf4fdde4876e8d76aa0131ad4d67c47b2b5591a/components/json/Get_element_by_key/component.yaml + name: Get element by key from JSON + author: Alexey Volkov + digest: 2790d7a0b9983e3b9ddca1fb02b243a7b3e0280198cd859438d622ad1ca8d3b0 + fetched_at: '2025-12-15T23:26:53.254Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/json/Query/component.yaml + name: Query JSON using JQ + author: Alexey Volkov + digest: 879a8ddf73f5bec05643901f88b024f82c83cf1443536b98a624331a89d8a9f5 + fetched_at: '2025-12-15T23:26:53.221Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/json/Build_dict/component.yaml + name: Build dict + author: Alexey Volkov + digest: a3d12264c5d5cd52900d2a8d16c96630954de1762c0000103c999a0d48396a7a + fetched_at: '2025-12-15T23:26:53.303Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/json/Build_list/component.yaml + name: Build list + author: Alexey Volkov + digest: 24b33123e1cb13712aec208e81e6c1dc2d6d643c8a1fe323581e8858c3c8993a + fetched_at: '2025-12-15T23:26:53.245Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/aecac18d4023c73c561d7f21192253e9593b9932/components/json/Build_list_of_strings/component.yaml + name: Build list of strings + author: Alexey Volkov + digest: 76963afe607da688f1334e8cff2017da891f92609855d6103ccbab627c6eb2ef + fetched_at: '2025-12-15T23:26:53.236Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/bb9d7518b3a23e945c8cc1663942063c6b92c20f/components/json/Build_list_of_integers/component.yaml + name: Build list of integers + author: Alexey Volkov + digest: 965f5a4f2f0a34f07419ab5d60cc3c3505ab2d7579baa0dd36d9a32ea97a7ffa + fetched_at: '2025-12-15T23:26:53.255Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/bb9d7518b3a23e945c8cc1663942063c6b92c20f/components/json/Build_list_of_floats/component.yaml + name: Build list of floats + author: Alexey Volkov + digest: b3ad1eeb79066b286809fe4a1b078aee08bbadd07d977b5327880e854bec1ff2 + fetched_at: '2025-12-15T23:26:53.233Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/json/Combine_lists/component.yaml + name: Combine lists + author: Alexey Volkov + digest: f2b024570631dd504413a7836297d02067423ff9d64b62ebc76be1a943488f4a + fetched_at: '2025-12-15T23:26:53.248Z' + - name: Upload/Download components: - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/f328b3f4842838b6578b701d975b22f7908aca48/components/web/Download/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/google-cloud/storage/download/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/6210648f30b2b3a8c01cc10be338da98300efb6b/components/google-cloud/storage/upload_to_unique_uri/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/6210648f30b2b3a8c01cc10be338da98300efb6b/components/google-cloud/storage/upload_to_explicit_uri/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/cca2d8569d01b527df10c629258be04d52eacc43/components/Download_and_upload/IPFS/Download/component.yaml" - + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/f328b3f4842838b6578b701d975b22f7908aca48/components/web/Download/component.yaml + name: Download data + author: Alexey Volkov + digest: 44c3cf1caa233e1d9057a25181ce622359643fc9345140b84556b00c8042ffb4 + fetched_at: '2025-12-15T23:26:53.223Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/google-cloud/storage/download/component.yaml + name: Download from GCS + author: Alexey Volkov + digest: 30c424ac6156c478aa0c3027b470baf9cb7dbbf90aebcabde7469bfbd02a512e + fetched_at: '2025-12-15T23:26:53.205Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/6210648f30b2b3a8c01cc10be338da98300efb6b/components/google-cloud/storage/upload_to_unique_uri/component.yaml + name: Upload to GCS with unique name + author: Alexey Volkov + digest: 074b5c68679117f5aeb41e36ee75b8c349ec88631ba552cbf6991bd5942d9192 + fetched_at: '2025-12-15T23:26:53.231Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/6210648f30b2b3a8c01cc10be338da98300efb6b/components/google-cloud/storage/upload_to_explicit_uri/component.yaml + name: Upload to GCS + author: Alexey Volkov + digest: e3cfa607ab9e2e0312ef4268dad157edf30c3d4ba5059b2e295fe047258aa31d + fetched_at: '2025-12-15T23:26:53.231Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/cca2d8569d01b527df10c629258be04d52eacc43/components/Download_and_upload/IPFS/Download/component.yaml + name: Download data from IPFS + author: Alexey Volkov + digest: 6fedde78a50fcece87df20eeba865d6f4a2c2513d514ff1386efe2e0d1445421 + fetched_at: '2025-12-15T23:26:53.283Z' - name: ML frameworks folders: - - name: "Scikit learn" + - name: Scikit learn components: - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml" - - - name: "XGBoost" + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml + name: Train linear regression model using scikit learn from CSV + author: Alexey Volkov + digest: 57ecdf4c3d54c5e6a032ba60667fea7af140d3568552bf5ea090510af5821a90 + fetched_at: '2025-12-15T23:26:53.250Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml + name: Train logistic regression model using scikit learn from CSV + author: Alexey Volkov + digest: 25410cf204a3a5d4274906541d5f0140c2257b719c0b98e07ab6e85b066e84e4 + fetched_at: '2025-12-15T23:26:53.303Z' + - name: XGBoost components: - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/XGBoost/Train/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/79d79efc5619692b28bcf14d37718077330ad617/components/XGBoost/Predict/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/XGBoost/Train/from_ApacheParquet/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/79d79efc5619692b28bcf14d37718077330ad617/components/XGBoost/Predict/from_ApacheParquet/component.yaml" - - - name: "PyTorch" + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/XGBoost/Train/component.yaml + name: Train XGBoost model on CSV + author: Alexey Volkov + digest: 5b8bec6716337d7bd8ecafd6dd46bc44527783bc05950cfe03a60206b43c7654 + fetched_at: '2025-12-15T23:26:53.275Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/79d79efc5619692b28bcf14d37718077330ad617/components/XGBoost/Predict/component.yaml + name: Xgboost predict on CSV + author: Alexey Volkov + digest: 86c68f66374a6cb0d0952fdd546fac65f750c9d1ed2578d9cb9094541bda9786 + fetched_at: '2025-12-15T23:26:53.248Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/XGBoost/Train/from_ApacheParquet/component.yaml + name: Train XGBoost model on ApacheParquet + author: Alexey Volkov + digest: eb6a6f2e21760995351eda950d9a350bf339410afb327eceab7df0618cceb656 + fetched_at: '2025-12-15T23:26:53.283Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/79d79efc5619692b28bcf14d37718077330ad617/components/XGBoost/Predict/from_ApacheParquet/component.yaml + name: Xgboost predict on ApacheParquet + author: Alexey Volkov + digest: 154892b1c81c3e401e73fe9c5bd2b79e841daf8289058c9ec986dcfd0267e6e7 + fetched_at: '2025-12-15T23:26:53.292Z' + - name: PyTorch components: - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/PyTorch/Create_fully_connected_network/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/7799eb6ae4c4bb817f7461e2e3ffc83cbe2d50a7/components/PyTorch/Predict/on_CSV/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d2b5b635198ce661c9c58e3a1817c4c018e7c40b/components/PyTorch/Train_PyTorch_model/from_ApacheParquet/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/7799eb6ae4c4bb817f7461e2e3ffc83cbe2d50a7/components/PyTorch/Predict/on_ApacheParquet/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Convert_to_OnnxModel_from_PyTorchScriptModule/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml" - - - name: "Tensorflow" + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/PyTorch/Create_fully_connected_network/component.yaml + name: Create fully connected pytorch network + author: Alexey Volkov + digest: 6d2718ef53bdd1fde52eefceb138b338b3a497585b344ae923040c70a560dde8 + fetched_at: '2025-12-15T23:26:53.289Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml + name: Train pytorch model from csv + author: Alexey Volkov + digest: 3c84cd7eddeda2d59ca84d37ea416bd78427aea8eeccd85005e958c203914b67 + fetched_at: '2025-12-15T23:26:53.256Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/7799eb6ae4c4bb817f7461e2e3ffc83cbe2d50a7/components/PyTorch/Predict/on_CSV/component.yaml + name: Predict with PyTorch model on CSV data + author: Alexey Volkov + digest: 40c4265e98ff35fa787bc35097a43df538ca43f7c7e802a990fc9c9e1d63e484 + fetched_at: '2025-12-15T23:26:53.292Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d2b5b635198ce661c9c58e3a1817c4c018e7c40b/components/PyTorch/Train_PyTorch_model/from_ApacheParquet/component.yaml + name: Train pytorch model from ApacheParquet + author: Alexey Volkov + digest: e906d1a7eb5636a68325a1ac5b481ddf5f3c4dbd6de51ce8143f67a3ac87ec35 + fetched_at: '2025-12-15T23:26:53.255Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/7799eb6ae4c4bb817f7461e2e3ffc83cbe2d50a7/components/PyTorch/Predict/on_ApacheParquet/component.yaml + name: Predict with PyTorch model on ApacheParquet data + author: Alexey Volkov + digest: 490205810db2510d850c672770ca2b15d2cc18b3e9336c2904ab0f5953edd58a + fetched_at: '2025-12-15T23:26:53.233Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Convert_to_OnnxModel_from_PyTorchScriptModule/component.yaml + name: Convert to onnx from pytorch script module + author: Alexey Volkov + digest: 7edca2cd8212000a52dedf6c60aeb808fd022818ccb0ee64ddc85259b84e6186 + fetched_at: '2025-12-15T23:26:53.289Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml + name: Create PyTorch Model Archive with base handler + author: Alexey Volkov + digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec + fetched_at: '2025-12-15T23:26:53.264Z' + - name: Tensorflow components: - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/tensorflow/Create_fully_connected_network/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/tensorflow/Train_model_using_Keras/on_ApacheParquet/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/tensorflow/Predict/on_CSV/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/tensorflow/Predict/on_ApacheParquet/component.yaml" - - - name: "CatBoost" + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/tensorflow/Create_fully_connected_network/component.yaml + name: Create fully connected tensorflow network + author: Alexey Volkov + digest: be1b5011e658a1b9f77fce7c5cce406fd0440064f17309b900c4116927cc3821 + fetched_at: '2025-12-15T23:26:53.289Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml + name: Train model using Keras on CSV + author: Alexey Volkov + digest: 2b0a50cae42e7469cf0add42eae88f66748e71a6826a087873b216aef3c687f9 + fetched_at: '2025-12-15T23:26:53.269Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/tensorflow/Train_model_using_Keras/on_ApacheParquet/component.yaml + name: Train model using Keras on ApacheParquet + author: Alexey Volkov + digest: 5f886b115e8c5e35c65e9f520b449b7cbcbdebb92114eeef2b7745497f6f6be1 + fetched_at: '2025-12-15T23:26:53.244Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/tensorflow/Predict/on_CSV/component.yaml + name: Predict with TensorFlow model on CSV data + author: Alexey Volkov + digest: db1fcabb3f542b203b59c799853be2d4d74253beb4e19d0eef3f5bf4b643222f + fetched_at: '2025-12-15T23:26:53.264Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/92aa941c738e5b2fe957f987925053bf70996264/components/tensorflow/Predict/on_ApacheParquet/component.yaml + name: Predict with TensorFlow model on ApacheParquet data + author: Alexey Volkov + digest: 7e760242a9c9ce013a367b866d3d1d7411986367bcfa59d847d853d3219f7fe8 + fetched_at: '2025-12-15T23:26:53.289Z' + - name: CatBoost components: - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/CatBoost/Train_regression/from_CSV/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/CatBoost/Train_classifier/from_CSV/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/CatBoost/Predict_values/from_CSV/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/CatBoost/Predict_classes/from_CSV/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/CatBoost/Predict_class_probabilities/from_CSV/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/CatBoost/convert_CatBoostModel_to_ONNX/component.yaml" - - - name: "Vowpal_Wabbit" + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/CatBoost/Train_regression/from_CSV/component.yaml + name: Catboost train regression + author: Alexey Volkov + digest: ac04356d7b08e9d3e90279befd4df446832641517efb34dea75354312347295a + fetched_at: '2025-12-15T23:26:53.250Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/CatBoost/Train_classifier/from_CSV/component.yaml + name: Catboost train classifier + author: Alexey Volkov + digest: f4ef9b474377248847c6487a827aa7b14d092a8d031c368168f2857a8ff11cad + fetched_at: '2025-12-15T23:26:53.266Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/CatBoost/Predict_values/from_CSV/component.yaml + name: Catboost predict values + author: Alexey Volkov + digest: 64f4651cd80d1bdbd524ff0cd2272916ebd3855f7b1f71d8abf474ccb0ec3dfb + fetched_at: '2025-12-15T23:26:53.248Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/CatBoost/Predict_classes/from_CSV/component.yaml + name: Catboost predict classes + author: Alexey Volkov + digest: 6e45f6a3b7a7805aa3c99a9d6c1bbf7434c365ad6f23de930014902bf19dc78f + fetched_at: '2025-12-15T23:26:53.290Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/CatBoost/Predict_class_probabilities/from_CSV/component.yaml + name: Catboost predict class probabilities + author: Alexey Volkov + digest: 9074952dc787000001b2f75291a287a1cc968648c711a8f648605fd0053999bb + fetched_at: '2025-12-15T23:26:53.243Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/CatBoost/convert_CatBoostModel_to_ONNX/component.yaml + name: Convert CatBoostModel to ONNX + author: Alexey Volkov + digest: f56b73b1e1e6b9e8745a4dfb49423ec33194adde2aac0b9d20d4c7bf77731907 + fetched_at: '2025-12-15T23:26:53.300Z' + - name: Vowpal_Wabbit components: - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/ML_frameworks/Vowpal_Wabbit/Create_JSON_dataset/from_CSV/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/a2a629e776d5fa0204ce71370cab23282d3e4278/components/ML_frameworks/Vowpal_Wabbit/Train_regression_model/from_VowpalWabbitJsonDataset/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/a2a629e776d5fa0204ce71370cab23282d3e4278/components/ML_frameworks/Vowpal_Wabbit/Predict/from_VowpalWabbitJsonDataset/component.yaml" - - # # Hidden: Need re-testing. Also, TFX is not popular. It's best to properly wrap individual libraries (TFDV, TFMA, etc). - # - name: TFX - # components: - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/deprecated/tfx/ExampleGen/CsvExampleGen/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/deprecated/tfx/StatisticsGen/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/deprecated/tfx/SchemaGen/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/deprecated/tfx/ExampleValidator/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/deprecated/tfx/Transform/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/deprecated/tfx/Trainer/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/deprecated/tfx/Evaluator/component.yaml" - - name: "ML Metrics" + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/ML_frameworks/Vowpal_Wabbit/Create_JSON_dataset/from_CSV/component.yaml + name: Create Vowpal Wabbit JSON dataset from CSV + author: Alexey Volkov + digest: 2e5d07f6df3fa21f7d0438136526d952a0c726302971c2e00ac42fe26856e674 + fetched_at: '2025-12-15T23:26:53.221Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/a2a629e776d5fa0204ce71370cab23282d3e4278/components/ML_frameworks/Vowpal_Wabbit/Train_regression_model/from_VowpalWabbitJsonDataset/component.yaml + name: Train regression model using Vowpal Wabbit on VowpalWabbitDataset + author: Alexey Volkov + digest: 601b68212b611b2b798767f4fcf3e77de662688b772daaae0ad3eb55426f56bc + fetched_at: '2025-12-15T23:26:53.248Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/a2a629e776d5fa0204ce71370cab23282d3e4278/components/ML_frameworks/Vowpal_Wabbit/Predict/from_VowpalWabbitJsonDataset/component.yaml + name: Predict using Vowpal Wabbit model on VowpalWabbitJsonDataset + author: Alexey Volkov + digest: f3eecb52c5f4d31d89090d17217e05f881600f8c7a18fc8de11eea1bf9d5fdf8 + fetched_at: '2025-12-15T23:26:53.197Z' + - name: ML Metrics components: - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/9b65c073ae0a131151454c7f9030cb1a553ca7ba/components/ml_metrics/Calculate_regression_metrics/from_CSV/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/89d9d83d77cdbbe521994f5d95893661e14f9172/components/ml_metrics/Calculate_regression_metrics/from_ApacheParquet/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/ml_metrics/Aggregate_regression_metrics/component.yaml" - - - name: "Converters" + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9b65c073ae0a131151454c7f9030cb1a553ca7ba/components/ml_metrics/Calculate_regression_metrics/from_CSV/component.yaml + name: Calculate regression metrics from CSV + author: Alexey Volkov + digest: e78a53973d5a627f65770d65e1a07659e878f131038ccb461e15b9b4edc7aa19 + fetched_at: '2025-12-15T23:26:53.289Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/89d9d83d77cdbbe521994f5d95893661e14f9172/components/ml_metrics/Calculate_regression_metrics/from_ApacheParquet/component.yaml + name: Calculate regression metrics from ApacheParquet + author: Alexey Volkov + digest: 2c46f26adb27c8bf4325304f11b9eb9d17b8b9d7c527683bfb38a2b70a6cbdd9 + fetched_at: '2025-12-15T23:26:53.303Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/ml_metrics/Aggregate_regression_metrics/component.yaml + name: Aggregate regression metrics + author: Alexey Volkov + digest: 1291e9c03fab7205acb9bb6b7b41920eed3b07ff3270f1dd57173e421d21f12e + fetched_at: '2025-12-15T23:26:53.300Z' + - name: Converters components: - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/_converters/ApacheParquet/from_CSV/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/_converters/ApacheParquet/from_TSV/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/_converters/ApacheParquet/from_ApacheArrowFeather/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/_converters/ApacheParquet/to_CSV/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/_converters/ApacheParquet/to_TSV/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/_converters/ApacheParquet/to_ApacheArrowFeather/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/4fca0aa607c00e60d5eb342630acc175e6f51fc2/components/_converters/XGBoostJsonModel/from_XGBoostModel/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/4fca0aa607c00e60d5eb342630acc175e6f51fc2/components/_converters/XGBoostJsonModel/to_XGBoostModel/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/4619f84eebc54230153ba48b0e67ac8446dc31c6/components/_converters/OnnxModel/from_XGBoostJsonModel/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/ae9d04e833d973d785809e7734021b06bcfea9bc/components/_converters/OnnxModel/from_XGBoostModel/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/_converters/OnnxModel/from_TensorflowSavedModel/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/2fc275072568cd0cdf73743ab49aa90928303f2c/components/_converters/OnnxModel/to_TensorflowSavedModel/component.yaml" - - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/f7f79decd10955b8408541f6f695a614538a6901/components/_converters/ScikitLearnPickleModel/to_OnnxModel/component.yaml" - - # # Hidden: Due to auth, these components only work out of the box on GCP (GKE or Vertex AI). But most of our deployments (local, HuggingFace) do not have that. - # - name: "Google Cloud" - # folders: - # - name: Vertex AI - # folders: - # - name: AutoML - # components: - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/47f3621344c884666a926c8a15d77562f1cc5e0a/components/google-cloud/Vertex_AI/AutoML/Tables/Create_dataset/from_CSV/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/00d020c29a144cee7fd35f2d05053addb942f536/components/google-cloud/Vertex_AI/AutoML/Tables/Create_dataset/from_GCS/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/00d020c29a144cee7fd35f2d05053addb942f536/components/google-cloud/Vertex_AI/AutoML/Tables/Create_dataset/from_BigQuery/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/ab85ecc9c30d4d68a2993ca87861f5e531a4f41b/components/google-cloud/Vertex_AI/AutoML/Tables/Train_model/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/a31b7c9652646f2cd035a0b3a23e0723c632521b/components/google-cloud/Vertex_AI/AutoML/Tables/Get_model_tuning_trials/component.yaml" - # - name: Models - # components: - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/2c24c0c0730c818b89f676c4dc5c9d6cb90ab01d/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/25dc317e649a19a53139a08ccbe496a248693fe4/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/719783ef44c04348ea23e247a93021d91cfe602d/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d1e7a3ccf8f8e0324e15922d6fd90d667fc5281b/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/b2cdd60fe93d609111729ef64e79a8b8a2713435/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d45e011ad8b62b4fe36c12289a624e5e1573c68d/components/google-cloud/Vertex_AI/Models/Export/to_GCS/component.yaml" - # - name: Storage - # components: - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/google-cloud/storage/download/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/6210648f30b2b3a8c01cc10be338da98300efb6b/components/google-cloud/storage/upload_to_explicit_uri/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/6210648f30b2b3a8c01cc10be338da98300efb6b/components/google-cloud/storage/upload_to_unique_uri/component.yaml" - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/google-cloud/storage/list/component.yaml" - # - name: AI Platform (legacy) - # folders: - # - name: Optimizer - # components: - # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/1b87c0bdfde5d7ec039401af8561783432731402/components/google-cloud/Optimizer/Suggest_parameter_sets_based_on_measurements/component.yaml" - # # # !!! Google Cloud AI Platform AutoML API cannot be called from Google Cloud Vertex AI Pipelines (~by design) - # # - name: AutoML - # # components: - # # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/3862f752bb825bb8cdcae842f0b294794028376b/components/google-cloud/AutoML/Tables/Create_dataset/from_CSV/component.yaml" - # # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/46b2a4ef4ac530404958839ae397b9fd533cf154/components/google-cloud/AutoML/Tables/Create_dataset/from_GCS/component.yaml" - # # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/247a0e0cf55c28b978c7f8e44fcedba356c12e93/components/google-cloud/AutoML/Tables/Create_dataset/from_BigQuery/component.yaml" - # # - url: "https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/gcp/automl/create_model_for_tables/component.yaml" + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/_converters/ApacheParquet/from_CSV/component.yaml + name: Convert csv to apache parquet + author: Alexey Volkov + digest: 01be9da9b7e2c1d63a72090cb1c47abf9a581b037c4a477584fbfdf1dcf85c86 + fetched_at: '2025-12-15T23:26:53.231Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/_converters/ApacheParquet/from_TSV/component.yaml + name: Convert tsv to apache parquet + author: Alexey Volkov + digest: 50b66cb4e967d5fd1ea3ca0e9de8956b1f313c9f3abb271ca426b050d4573eac + fetched_at: '2025-12-15T23:26:53.264Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/_converters/ApacheParquet/from_ApacheArrowFeather/component.yaml + name: Convert apache arrow feather to apache parquet + author: Alexey Volkov + digest: 9ad7aee6ca2fb82841c30cf2995b75c063f19b1f9c935bda1a9ef3d8ac1efed6 + fetched_at: '2025-12-15T23:26:53.277Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/_converters/ApacheParquet/to_CSV/component.yaml + name: Convert apache parquet to csv + author: Alexey Volkov + digest: fc9d4d0537d59e61dae838cbce8bbc58a338ea5e87eb504f941ef35965e08cc9 + fetched_at: '2025-12-15T23:26:53.232Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/_converters/ApacheParquet/to_TSV/component.yaml + name: Convert apache parquet to tsv + author: Alexey Volkov + digest: 7812764bbc6b9bdd4d71e406dea9575943c084ba7799cafd7e378085d76d4c42 + fetched_at: '2025-12-15T23:26:53.243Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/96a177dd71d54c98573c4101d9a05ac801f1fa54/components/_converters/ApacheParquet/to_ApacheArrowFeather/component.yaml + name: Convert apache parquet to apache arrow feather + author: Alexey Volkov + digest: 53a6051a8cac9ce5eb5fe0950c40c48b5ec78ecddbe76bbafdeffbc586f280a6 + fetched_at: '2025-12-15T23:26:53.250Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4fca0aa607c00e60d5eb342630acc175e6f51fc2/components/_converters/XGBoostJsonModel/from_XGBoostModel/component.yaml + name: Convert to XGBoostJsonModel from XGBoostModel + author: Alexey Volkov + digest: f82823ba99ab6fc6696e37685a00918aa94a9186b0a1979206b86ca3818099a4 + fetched_at: '2025-12-15T23:26:53.271Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4fca0aa607c00e60d5eb342630acc175e6f51fc2/components/_converters/XGBoostJsonModel/to_XGBoostModel/component.yaml + name: Convert to XGBoostModel from XGBoostJsonModel + author: Alexey Volkov + digest: b45de01b867736c16b2a68f53aa808c02f8e0caa2bffe2172aad7a20cdd17063 + fetched_at: '2025-12-15T23:26:53.261Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4619f84eebc54230153ba48b0e67ac8446dc31c6/components/_converters/OnnxModel/from_XGBoostJsonModel/component.yaml + name: Convert to OnnxModel from XGBoostJsonModel + author: Alexey Volkov + digest: 5e02e5fa2f12181a6b9588b152e9ee78d4952fea7f3d5740dfdbd15b3b2c0ee9 + fetched_at: '2025-12-15T23:26:53.217Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/ae9d04e833d973d785809e7734021b06bcfea9bc/components/_converters/OnnxModel/from_XGBoostModel/component.yaml + name: Convert to OnnxModel from XGBoostModel + author: Alexey Volkov + digest: 3cdc24652d1ab9b436dbe935d291cd03a1d6e14c039f8fc6e67e25ff6f80b0e5 + fetched_at: '2025-12-15T23:26:53.292Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/_converters/OnnxModel/from_TensorflowSavedModel/component.yaml + name: To ONNX from Tensorflow SavedModel + author: Alexey Volkov + digest: d44f6b94387edfc6560bf452c5662de431d2e1141b670e7ab9e921eba25396a8 + fetched_at: '2025-12-15T23:26:53.277Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/2fc275072568cd0cdf73743ab49aa90928303f2c/components/_converters/OnnxModel/to_TensorflowSavedModel/component.yaml + name: Convert to TensorflowSavedModel from OnnxModel + author: Alexey Volkov + digest: 93f892f4f2bdace722e473cae22141715b6328bdc0a508cd6878f050449ff92d + fetched_at: '2025-12-15T23:26:53.280Z' + - url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/f7f79decd10955b8408541f6f695a614538a6901/components/_converters/ScikitLearnPickleModel/to_OnnxModel/component.yaml + name: Convert to OnnxModel from ScikitLearnPickleModel + author: Alexey Volkov + digest: 1899985bd972ba5849be119fc841633e940f293f2314ae52d5cdd9ace2fda15f + fetched_at: '2025-12-15T23:26:53.240Z' +metadata: + augmented_at: '2025-12-15T23:26:53.389Z' + augmented_by: augment-component-library script diff --git a/scripts/hydrate-component-library.ts b/scripts/hydrate-component-library.ts new file mode 100644 index 000000000..65526a77c --- /dev/null +++ b/scripts/hydrate-component-library.ts @@ -0,0 +1,281 @@ +#!/usr/bin/env node + +import crypto from "crypto"; +import fs from "fs/promises"; +import http from "http"; +import https from "https"; +import yaml from "js-yaml"; +import process from "process"; + +// Type definitions +interface ComponentSpec { + name?: string; + metadata?: { + annotations?: { + author?: string; + [key: string]: unknown; + }; + [key: string]: unknown; + }; + [key: string]: unknown; +} + +interface BaseComponent { + url?: string; + [key: string]: unknown; +} + +interface ProcessedComponent extends BaseComponent { + name?: string | null; + author?: string | null; + digest?: string; + fetched_at?: string; + error?: string; +} + +interface Folder { + components?: BaseComponent[]; + folders?: Folder[]; + [key: string]: unknown; +} + +interface ProcessedFolder extends Omit { + components?: ProcessedComponent[]; + folders?: ProcessedFolder[]; +} + +interface ComponentLibrary { + folders: Folder[]; + metadata?: Record; + [key: string]: unknown; +} + +interface HydratedComponentLibrary extends Omit { + folders: ProcessedFolder[]; + metadata: { + augmented_at?: string; + augmented_by?: string; + [key: string]: unknown; + }; +} + +/** + * Generate a SHA-256 digest for a given text + * @param text - The text to hash + * @returns The hex-encoded digest + */ +async function generateDigest(text: string): Promise { + const hashBuffer = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(text), + ); + + return Array.from(new Uint8Array(hashBuffer)) + .map((b) => b.toString(16).padStart(2, "0")) + .join(""); +} + +/** + * Download content from a URL + * @param url - The URL to download from + * @returns The downloaded content + */ +function downloadContent(url: string): Promise { + return new Promise((resolve, reject) => { + const protocol = url.startsWith("https") ? https : http; + + protocol + .get(url, (res) => { + if (res.statusCode !== 200) { + reject( + new Error( + `Failed to download ${url}: ${res.statusCode} ${res.statusMessage}`, + ), + ); + return; + } + + let data = ""; + res.on("data", (chunk: Buffer | string) => { + data += chunk; + }); + + res.on("end", () => { + resolve(data); + }); + + res.on("error", reject); + }) + .on("error", reject); + }); +} + +function isValidComponentSpec(spec: unknown): spec is ComponentSpec { + return typeof spec === "object" && spec !== null; +} + +function parseComponentSpec(content: string): ComponentSpec { + const spec = yaml.load(content); + if (!isValidComponentSpec(spec)) { + throw new Error("Invalid component specification format"); + } + return spec; +} + +/** + * Process a single component by downloading and extracting metadata + * @param component - The component object with URL + * @returns The augmented component + */ +async function processComponent( + component: BaseComponent, +): Promise { + if (!component.url) { + return component; + } + + try { + console.log(`Processing: ${component.url}`); + + // Download the component YAML + const content = await downloadContent(component.url); + + // Parse the YAML to get ComponentSpec + const componentSpec = parseComponentSpec(content); + + // Extract fields + const name = componentSpec.name || null; + const author = componentSpec.metadata?.annotations?.author || null; + const digest = await generateDigest(content); + const fetched_at = new Date().toISOString(); + + // Return augmented component + return { + ...component, + name, + author, + digest, + fetched_at, + }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + console.error(`Error processing ${component.url}:`, errorMessage); + // Return original component with error info + return { + ...component, + error: errorMessage, + fetched_at: new Date().toISOString(), + }; + } +} + +/** + * Process a folder recursively + * @param folder - The folder object + * @returns The processed folder + */ +async function processFolder(folder: Folder): Promise { + const processedFolder: ProcessedFolder = { + ...folder, + components: undefined, + folders: undefined, + }; + + // Process components in this folder + if (folder.components && Array.isArray(folder.components)) { + processedFolder.components = await Promise.all( + folder.components.map(processComponent), + ); + } + + // Process subfolders recursively + if (folder.folders && Array.isArray(folder.folders)) { + processedFolder.folders = await Promise.all( + folder.folders.map(processFolder), + ); + } + + return processedFolder; +} + +/** + * Count components recursively for statistics + * @param folder - The folder to count components in + * @param stats - Statistics accumulator + */ +function countComponents( + folder: ProcessedFolder, + stats: { total: number; successful: number; failed: number }, +): void { + if (folder.components) { + folder.components.forEach((comp) => { + stats.total++; + if (comp.error) { + stats.failed++; + } else if (comp.digest) { + stats.successful++; + } + }); + } + if (folder.folders) { + folder.folders.forEach((subfolder) => countComponents(subfolder, stats)); + } +} + +/** + * Main function to process the component library + */ +async function main(): Promise { + try { + // Parse command line arguments + const args = process.argv.slice(2); + const inputFile = args[0] || "public/component_library.yaml"; + const outputFile = args[1] || "public/component_library_augmented.yaml"; + + console.log(`Reading component library from: ${inputFile}`); + + // Read the component library file + const libraryContent = await fs.readFile(inputFile, "utf8"); + const library = yaml.load(libraryContent) as ComponentLibrary; + + console.log("Processing component library..."); + + // Process all folders + const augmentedLibrary: HydratedComponentLibrary = { + ...library, + folders: await Promise.all(library.folders.map(processFolder)), + metadata: { + ...library.metadata, + augmented_at: new Date().toISOString(), + augmented_by: "augment-component-library script", + }, + }; + + // Write the augmented library + const outputContent = yaml.dump(augmentedLibrary, { + lineWidth: -1, + noRefs: true, + sortKeys: false, + }); + + await fs.writeFile(outputFile, outputContent, "utf8"); + console.log(`Augmented component library saved to: ${outputFile}`); + + // Print summary + const stats = { total: 0, successful: 0, failed: 0 }; + augmentedLibrary.folders.forEach((folder) => + countComponents(folder, stats), + ); + + console.log("\nSummary:"); + console.log(`Total components: ${stats.total}`); + console.log(`Successfully processed: ${stats.successful}`); + console.log(`Failed: ${stats.failed}`); + } catch (error) { + console.error("Error:", error); + process.exit(1); + } +} + +// Run the main function +void main();