diff --git a/.gitignore b/.gitignore index 0704a0eb..7156a1fc 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,4 @@ node_modules .rustup /config/ local_data -.env \ No newline at end of file +.env diff --git a/Cargo.lock b/Cargo.lock index c6a6af89..ba3cc583 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,7 +91,7 @@ checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" [[package]] name = "async-bb8-diesel" version = "0.3.0" -source = "git+https://github.com/oxidecomputer/async-bb8-diesel#18baf49aca5f72bf441951b11669c6c3c3affe70" +source = "git+https://github.com/oxidecomputer/async-bb8-diesel#f049570ff89080b2385c637af4c0e614b0a124a4" dependencies = [ "async-trait", "bb8", @@ -157,6 +157,453 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-config" +version = "1.8.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c456581cb3c77fafcc8c67204a70680d40b61112d6da78c77bd31d945b65f1b5" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 1.4.0", + "ring", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd362783681b15d136480ad555a099e82ecd8e2d10a841e14dfd0078d67fee3" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-lc-rs" +version = "1.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b7b6141e96a8c160799cc2d5adecd5cbbe5054cb8c7c4af53da0f83bb7ad256" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c34dda4df7017c8db52132f0f8a2e0f8161649d15723ed63fc00c82d0f2081a" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "aws-runtime" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c635c2dc792cb4a11ce1a4f392a925340d1bdf499289b5ec1ec6810954eb43f5" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.122.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94c2ca0cba97e8e279eb6c0b2d0aa10db5959000e602ab2b7c02de6b85d4c19b" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.0", + "http-body 1.0.1", + "lru", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.93.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dcb38bb33fc0a11f1ffc3e3e85669e0a11a37690b86f77e75306d8f369146a0" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.95.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ada8ffbea7bd1be1f53df1dadb0f8fdb04badb13185b3321b929d1ee3caad09" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.97.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6443ccadc777095d5ed13e21f5c364878c9f5bad4e35187a6cdbd863b0afcad" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efa49f3c607b92daae0c078d48a4571f599f966dce3caee5f1ea55c4d9073f99" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.0", + "p256 0.11.1", + "percent-encoding", + "ring", + "sha2", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52eec3db979d18cb807fc1070961cc51d87d069abe9ab57917769687368a8c6c" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.64.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcf418858f9f3edd228acb8759d77394fed7531cce78d02bdda499025368439" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc-fast", + "hex", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35b9c7354a3b13c66f60fe4616d6d1969c9fd36b1b5333a5dfb3ee716b33c588" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.63.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630e67f2a31094ffa51b210ae030855cb8f3b7ee1329bdd8d085aaf61e8b97fc" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12fb0abf49ff0cab20fd31ac1215ed7ce0ea92286ba09e2854b42ba5cabe7525" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.3.27", + "h2 0.4.13", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper 1.8.1", + "hyper-rustls 0.24.2", + "hyper-rustls 0.27.7", + "hyper-util", + "pin-project-lite", + "rustls 0.21.12", + "rustls 0.23.36", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.4", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.62.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cb96aa208d62ee94104645f7b2ecaf77bf27edf161590b6224bfbac2832f979" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0a46543fbc94621080b3cf553eb4cbbdc41dd9780a30c4756400f0139440a1d" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cebbddb6f3a5bd81553643e9c7daf3cc3dc5b0b5f398ac668630e8a84e6fff0" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3df87c14f0127a0d77eb261c3bc45d5b4833e2a1f63583ebfb728e4852134ee" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49952c52f7eebb72ce2a754d3866cc0f87b97d2a46146b79f80f3a93fb2b3716" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.4.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3a26048eeab0ddeba4b4f9d51654c79af8c3b32357dc5f336cee85ab331c33" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11b2f670422ff42bf7065031e72b45bc52a3508bd089f743ea90731ca2b6ea57" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d980627d2dd7bfc32a3c025685a033eeab8d365cc840c631ef59d1b8f428164" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base16ct" version = "0.2.0" @@ -169,6 +616,16 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.8.3" @@ -225,13 +682,23 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" dependencies = [ "serde", ] +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "camino" version = "1.2.2" @@ -243,11 +710,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.54" +version = "1.2.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6354c81bbfd62d9cfa9cb3c773c2b7b2a3a482d569de977fd0e961f6e7c00583" +checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -279,9 +748,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.54" +version = "4.5.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394" +checksum = "6899ea499e3fb9305a65d5ebf6e3d2248c5fab291f300ad0a704fbe142eae31a" dependencies = [ "clap_builder", "clap_derive", @@ -289,9 +758,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.54" +version = "4.5.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00" +checksum = "7b12c8b680195a62a8364d16b8447b01b6c2c8f9aaf68bee653be34d4245e238" dependencies = [ "anstream", "anstyle", @@ -301,9 +770,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.49" +version = "4.5.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" dependencies = [ "heck", "proc-macro2", @@ -317,6 +786,15 @@ version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32" +[[package]] +name = "cmake" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.4" @@ -423,6 +901,33 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc-fast" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd92aca2c6001b1bf5ba0ff84ee74ec8501b52bbef0cac80bf25a6c1d87a83d" +dependencies = [ + "crc", + "digest", + "rustversion", + "spin 0.10.0", +] + [[package]] name = "crc32c" version = "0.6.8" @@ -432,6 +937,15 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -451,7 +965,19 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" name = "crunchy" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] [[package]] name = "crypto-bigint" @@ -557,6 +1083,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffe7ed1d93f4553003e20b629abe9085e1e81b1429520f897f8f8860bc6dfc21" +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der" version = "0.7.10" @@ -716,9 +1252,9 @@ checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" [[package]] name = "dropshot" -version = "0.16.6" +version = "0.16.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d0df98c06659ab85a454f32dc36ca5dbc6500bd2a58f25ede4dc1f1d478904e" +checksum = "d69fd85c8dfc67252d02f260595f6b62b5abceb1b88b4b9722369d27936e5fa4" dependencies = [ "async-stream", "async-trait", @@ -733,7 +1269,7 @@ dependencies = [ "hostname 0.4.2", "http 1.4.0", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-util", "indexmap 2.13.0", "multer", @@ -767,7 +1303,7 @@ dependencies = [ [[package]] name = "dropshot-authorization-header" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/v-api#5ea039de839c4a88943961e8bbfb298781671755" +source = "git+https://github.com/oxidecomputer/v-api#21d760b210429e23df76a1f6bf691c94b3de5488" dependencies = [ "async-trait", "base64", @@ -786,7 +1322,7 @@ dependencies = [ "hex", "hmac", "http 1.4.0", - "hyper", + "hyper 1.8.1", "schemars 0.8.22", "serde", "serde_json", @@ -797,9 +1333,9 @@ dependencies = [ [[package]] name = "dropshot_endpoint" -version = "0.16.6" +version = "0.16.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e53aef8838e0e341485590738ab180a6dceff3565ffcb198d5f365fea650378" +checksum = "67d106478e4a4782556981d028a667f41c4845cdaa6e2d3a9f58c5d15e725401" dependencies = [ "heck", "proc-macro2", @@ -824,24 +1360,42 @@ dependencies = [ "syn", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der 0.6.1", + "elliptic-curve 0.12.3", + "rfc6979 0.3.1", + "signature 1.6.4", +] + [[package]] name = "ecdsa" version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der", + "der 0.7.10", "digest", - "elliptic-curve", - "rfc6979", - "signature", - "spki", + "elliptic-curve 0.13.8", + "rfc6979 0.4.0", + "signature 2.2.0", + "spki 0.7.3", ] [[package]] @@ -859,8 +1413,8 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", - "signature", + "pkcs8 0.10.2", + "signature 2.2.0", ] [[package]] @@ -886,23 +1440,43 @@ dependencies = [ "serde", ] +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct 0.1.1", + "crypto-bigint 0.4.9", + "der 0.6.1", + "digest", + "ff 0.12.1", + "generic-array", + "group 0.12.1", + "pkcs8 0.9.0", + "rand_core 0.6.4", + "sec1 0.3.0", + "subtle", + "zeroize", +] + [[package]] name = "elliptic-curve" version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ - "base16ct", - "crypto-bigint", + "base16ct 0.2.0", + "crypto-bigint 0.5.5", "digest", - "ff", + "ff 0.13.1", "generic-array", - "group", + "group 0.13.0", "hkdf", "pem-rfc7468", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "sec1", + "sec1 0.7.3", "subtle", "zeroize", ] @@ -958,6 +1532,16 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "ff" version = "0.13.1" @@ -976,9 +1560,9 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "find-msvc-tools" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "fnv" @@ -1013,6 +1597,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619" +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.31" @@ -1156,7 +1746,7 @@ dependencies = [ "chrono", "http 1.4.0", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-util", "mime", "percent-encoding", @@ -1176,8 +1766,8 @@ checksum = "cef55117f2b1d40e56f2fd26161a2e93c9d7899be9389a9d4aa59308168e880c" dependencies = [ "chrono", "google-apis-common", - "hyper", - "hyper-rustls", + "hyper 1.8.1", + "hyper-rustls 0.27.7", "hyper-util", "mime", "serde", @@ -1196,8 +1786,8 @@ checksum = "30742c5730cad187ffb8922f1a91766830676ec6b714489fcdb89e9b9385fd7c" dependencies = [ "chrono", "google-apis-common", - "hyper", - "hyper-rustls", + "hyper 1.8.1", + "hyper-rustls 0.27.7", "hyper-util", "mime", "serde", @@ -1216,8 +1806,8 @@ checksum = "0d454a68994bbdae724ec6a3129a23ec2a86ef29f8ae9e92bb3a3578c193284c" dependencies = [ "chrono", "google-apis-common", - "hyper", - "hyper-rustls", + "hyper 1.8.1", + "hyper-rustls 0.27.7", "hyper-util", "mime", "serde", @@ -1251,17 +1841,47 @@ dependencies = [ "web-time", ] +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff 0.12.1", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "group" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ - "ff", + "ff 0.13.1", "rand_core 0.6.4", "subtle", ] +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.13.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.13" @@ -1410,6 +2030,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -1429,7 +2060,7 @@ dependencies = [ "bytes", "futures-core", "http 1.4.0", - "http-body", + "http-body 1.0.1", "pin-project-lite", ] @@ -1445,6 +2076,30 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.8.1" @@ -1455,9 +2110,9 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", + "h2 0.4.13", "http 1.4.0", - "http-body", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -1468,6 +2123,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.7" @@ -1475,7 +2145,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ "http 1.4.0", - "hyper", + "hyper 1.8.1", "hyper-util", "rustls 0.23.36", "rustls-native-certs", @@ -1488,23 +2158,22 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ "base64", "bytes", "futures-channel", - "futures-core", "futures-util", "http 1.4.0", - "http-body", - "hyper", + "http-body 1.0.1", + "hyper 1.8.1", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.2", "system-configuration", "tokio", "tower-layer", @@ -1515,9 +2184,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.64" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1737,6 +2406,16 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.85" @@ -1775,16 +2454,16 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "10.2.0" +version = "10.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c76e1c7d7df3e34443b3621b459b066a7b79644f059fc8b2db7070c825fd417e" +checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1" dependencies = [ "base64", "ed25519-dalek", "getrandom 0.2.17", "hmac", "js-sys", - "p256", + "p256 0.13.2", "p384", "pem", "rand 0.8.5", @@ -1792,7 +2471,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "signature", + "signature 2.2.0", "simple_asn1", ] @@ -1802,7 +2481,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin", + "spin 0.9.8", ] [[package]] @@ -1857,6 +2536,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "lru" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -1996,9 +2684,9 @@ dependencies = [ [[package]] name = "minijinja" -version = "2.14.0" +version = "2.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ea9ac0a51fb5112607099560fdf0f90366ab088a2a9e6e8ae176794e9806aa" +checksum = "b479616bb6f0779fb0f3964246beda02d4b01144e1b0d5519616e012ccc2a245" dependencies = [ "memo-map", "self_cell", @@ -2055,7 +2743,7 @@ dependencies = [ "httparse", "memchr", "mime", - "spin", + "spin 0.9.8", "version_check", ] @@ -2274,20 +2962,37 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "owo-colors" version = "4.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c6901729fa79e91a0913333229e9ca5dc725089d1c363b2f4b4760709dc4a52" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa 0.14.8", + "elliptic-curve 0.12.3", + "sha2", +] + [[package]] name = "p256" version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ - "ecdsa", - "elliptic-curve", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", "primeorder", "sha2", ] @@ -2298,8 +3003,8 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" dependencies = [ - "ecdsa", - "elliptic-curve", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", "primeorder", "sha2", ] @@ -2480,9 +3185,19 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", - "pkcs8", - "spki", + "der 0.7.10", + "pkcs8 0.10.2", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der 0.6.1", + "spki 0.6.0", ] [[package]] @@ -2491,8 +3206,8 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", ] [[package]] @@ -2503,9 +3218,9 @@ checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "portable-atomic" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "potential_utf" @@ -2574,7 +3289,7 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" dependencies = [ - "elliptic-curve", + "elliptic-curve 0.13.8", ] [[package]] @@ -2680,7 +3395,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.23.36", - "socket2", + "socket2 0.6.2", "thiserror 2.0.18", "tokio", "tracing", @@ -2717,7 +3432,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.6.2", "tracing", "windows-sys 0.60.2", ] @@ -2878,9 +3593,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.2" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", @@ -2890,20 +3605,26 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" [[package]] name = "regress" @@ -2925,12 +3646,12 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "h2", + "h2 0.4.13", "http 1.4.0", - "http-body", + "http-body 1.0.1", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.8.1", + "hyper-rustls 0.27.7", "hyper-util", "js-sys", "log", @@ -2996,7 +3717,7 @@ dependencies = [ "futures", "getrandom 0.2.17", "http 1.4.0", - "hyper", + "hyper 1.8.1", "parking_lot 0.11.2", "reqwest", "reqwest-middleware", @@ -3049,6 +3770,17 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -3078,8 +3810,8 @@ dependencies = [ "dropshot-verified-body", "hex", "http 1.4.0", - "hyper", - "jsonwebtoken 10.2.0", + "hyper 1.8.1", + "jsonwebtoken 10.3.0", "meilisearch-sdk", "minijinja", "mockall", @@ -3136,7 +3868,7 @@ dependencies = [ "dirs 6.0.0", "futures", "itertools", - "jsonwebtoken 10.2.0", + "jsonwebtoken 10.3.0", "oauth2", "owo-colors", "progenitor-client", @@ -3217,6 +3949,8 @@ name = "rfd-processor" version = "0.12.3" dependencies = [ "async-trait", + "aws-config", + "aws-sdk-s3", "base64", "chrono", "config", @@ -3230,6 +3964,7 @@ dependencies = [ "md-5", "meilisearch-sdk", "mime_guess", + "mockall", "newtype-uuid", "octorust", "parse-rfd", @@ -3307,11 +4042,11 @@ dependencies = [ "num-integer", "num-traits", "pkcs1", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", "sha2", - "signature", - "spki", + "signature 2.2.0", + "spki 0.7.3", "subtle", "zeroize", ] @@ -3367,6 +4102,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.22.4" @@ -3387,6 +4134,7 @@ version = "0.23.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" dependencies = [ + "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", @@ -3426,6 +4174,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.102.8" @@ -3443,6 +4201,7 @@ version = "0.103.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -3517,9 +4276,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54e910108742c57a770f492731f99be216a52fadd361b06c8fb59d74ccc267d2" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" dependencies = [ "dyn-clone", "ref-cast", @@ -3545,22 +4304,46 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "seahash" version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct 0.1.1", + "der 0.6.1", + "generic-array", + "pkcs8 0.9.0", + "subtle", + "zeroize", +] + [[package]] name = "sec1" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "base16ct", - "der", + "base16ct 0.2.0", + "der 0.7.10", "generic-array", - "pkcs8", + "pkcs8 0.10.2", "subtle", "zeroize", ] @@ -3745,7 +4528,7 @@ dependencies = [ "indexmap 1.9.3", "indexmap 2.13.0", "schemars 0.9.0", - "schemars 1.2.0", + "schemars 1.2.1", "serde_core", "serde_json", "serde_with_macros", @@ -3824,6 +4607,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "signature" version = "2.2.0" @@ -3854,9 +4647,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "slog" @@ -3926,6 +4719,16 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.2" @@ -3942,6 +4745,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + [[package]] name = "spinning_top" version = "0.3.0" @@ -3951,6 +4760,16 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der 0.6.1", +] + [[package]] name = "spki" version = "0.7.3" @@ -3958,7 +4777,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", ] [[package]] @@ -4035,9 +4854,9 @@ dependencies = [ [[package]] name = "system-configuration" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ "bitflags 2.10.0", "core-foundation 0.9.4", @@ -4231,7 +5050,7 @@ dependencies = [ "parking_lot 0.12.5", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.6.2", "tokio-macros", "windows-sys 0.61.2", ] @@ -4247,6 +5066,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.25.0" @@ -4399,7 +5228,7 @@ dependencies = [ "bytes", "futures-util", "http 1.4.0", - "http-body", + "http-body 1.0.1", "iri-string", "pin-project-lite", "tower", @@ -4646,6 +5475,12 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -4673,7 +5508,7 @@ dependencies = [ [[package]] name = "v-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/v-api#5ea039de839c4a88943961e8bbfb298781671755" +source = "git+https://github.com/oxidecomputer/v-api#21d760b210429e23df76a1f6bf691c94b3de5488" dependencies = [ "async-trait", "base64", @@ -4688,8 +5523,8 @@ dependencies = [ "hex", "http 1.4.0", "http-body-util", - "hyper", - "jsonwebtoken 10.2.0", + "hyper 1.8.1", + "jsonwebtoken 10.3.0", "newtype-uuid", "oauth2", "partial-struct", @@ -4718,7 +5553,7 @@ dependencies = [ [[package]] name = "v-api-installer" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/v-api#5ea039de839c4a88943961e8bbfb298781671755" +source = "git+https://github.com/oxidecomputer/v-api#21d760b210429e23df76a1f6bf691c94b3de5488" dependencies = [ "diesel", "diesel_migrations", @@ -4727,7 +5562,7 @@ dependencies = [ [[package]] name = "v-api-permission-derive" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/v-api#5ea039de839c4a88943961e8bbfb298781671755" +source = "git+https://github.com/oxidecomputer/v-api#21d760b210429e23df76a1f6bf691c94b3de5488" dependencies = [ "heck", "proc-macro2", @@ -4738,7 +5573,7 @@ dependencies = [ [[package]] name = "v-model" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/v-api#5ea039de839c4a88943961e8bbfb298781671755" +source = "git+https://github.com/oxidecomputer/v-api#21d760b210429e23df76a1f6bf691c94b3de5488" dependencies = [ "async-bb8-diesel", "async-trait", @@ -4773,6 +5608,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "waitgroup" version = "0.1.2" @@ -4925,9 +5766,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" dependencies = [ "rustls-pki-types", ] @@ -5210,6 +6051,12 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "xtask" version = "0.0.0" @@ -5280,8 +6127,8 @@ dependencies = [ "base64", "http 1.4.0", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.8.1", + "hyper-rustls 0.27.7", "hyper-util", "log", "percent-encoding", @@ -5297,18 +6144,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.34" +version = "0.8.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71ddd76bcebeed25db614f82bf31a9f4222d3fbba300e6fb6c00afa26cbd4d9d" +checksum = "57cf3aa6855b23711ee9852dfc97dfaa51c45feaba5b645d0c777414d494a961" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.34" +version = "0.8.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8187381b52e32220d50b255276aa16a084ec0a9017a0ca2152a1f55c539758d" +checksum = "8a616990af1a287837c4fe6596ad77ef57948f787e46ce28e166facc0cc1cb75" dependencies = [ "proc-macro2", "quote", @@ -5377,6 +6224,6 @@ dependencies = [ [[package]] name = "zmij" -version = "1.0.17" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02aae0f83f69aafc94776e879363e9771d7ecbffe2c7fbb6c14c5e00dfe88439" +checksum = "3ff05f8caa9038894637571ae6b9e29466c1f4f829d26c9b28f869a29cbe3445" diff --git a/Cargo.toml b/Cargo.toml index b74c6c61..538a7872 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,8 @@ resolver = "2" anyhow = "1.0.100" async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", version = "0.3" } async-trait = "0.1.89" +aws-config = "1.8.13" +aws-sdk-s3 = "1.99.0" base64 = "0.22" bb8 = "0.9" chrono = "0.4.43" diff --git a/Containerfile b/Containerfile new file mode 100644 index 00000000..d31ed052 --- /dev/null +++ b/Containerfile @@ -0,0 +1,86 @@ +# syntax=docker/dockerfile:1 +# Build stage +FROM docker.io/rust:1-trixie AS builder + +ARG DIESEL_VERSION=v2.3.5 +ARG NODE_VERSION=v24.13.0 + +WORKDIR /app + +# Install build dependencies for diesel/postgres +RUN apt-get update && apt-get install -y \ + libpq-dev \ + pkg-config + +# Copy workspace files +COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ + +# Copy all workspace members +COPY parse-rfd ./parse-rfd +COPY rfd-api ./rfd-api +COPY rfd-cli ./rfd-cli +COPY rfd-data ./rfd-data +COPY rfd-github ./rfd-github +COPY rfd-installer ./rfd-installer +COPY rfd-model ./rfd-model +COPY rfd-processor ./rfd-processor +COPY rfd-sdk ./rfd-sdk +COPY trace-request ./trace-request +COPY xtask ./xtask + +ENV CARGO_HOME=/data/cargo + +# Build all target binaries in release mode +RUN cargo build --release \ + --package rfd-api \ + --package rfd-processor \ + --package rfd-cli \ + --package rfd-installer + +# Download diesel tool for migrations +WORKDIR /tmp +RUN curl -L --output-dir /tmp -O "https://github.com/diesel-rs/diesel/releases/download/${DIESEL_VERSION}/diesel_cli-x86_64-unknown-linux-gnu.tar.xz" \ + && curl -L --output-dir /tmp -O "https://github.com/diesel-rs/diesel/releases/download/${DIESEL_VERSION}/diesel_cli-x86_64-unknown-linux-gnu.tar.xz.sha256" \ + && sha256sum diesel_cli-x86_64-unknown-linux-gnu.tar.xz.sha256 && tar --strip-components=1 -xJvf diesel_cli-x86_64-unknown-linux-gnu.tar.xz + +# Node for search indec +RUN curl -L --output-dir /tmp -O "https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-x64.tar.xz" \ + && curl -L --output-dir /tmp -o node.SHASUMS256.txt "https://nodejs.org/dist/${NODE_VERSION}/SHASUMS256.txt" \ + && sha256sum node.SHASUMS256.txt && tar --strip-components=1 -xJvf "node-${NODE_VERSION}-linux-x64.tar.xz" + +# Runtime stage +FROM docker.io/debian:trixie-slim + +# Install runtime dependencies and tini +RUN apt-get update && apt-get install -y \ + ca-certificates \ + libpq5 \ + tini \ + && rm -rf /var/lib/apt/lists/* + +RUN useradd --create-home --user-group rfd \ + && mkdir /home/rfd/db + +# Copy binaries from builder +COPY --from=builder /app/target/release/rfd-api /usr/local/bin/ +COPY --from=builder /app/target/release/rfd-processor /usr/local/bin/ +COPY --from=builder /app/target/release/rfd-cli /usr/local/bin/ +COPY --from=builder /app/target/release/rfd-installer /usr/local/bin/ + +# Database migrations for diesel +COPY --from=builder /tmp/diesel /usr/local/bin/ +# COPY --from=builder /app/rfd-model/diesel.toml /home/rfd/db/ +COPY --from=builder /app/rfd-model/migrations/ /home/rfd/db/migrations/ + +# Node for search indexing +COPY --from=builder /tmp/bin/node /usr/local/bin/ + +# Create non-root user +USER rfd +WORKDIR /home/rfd + +# Use tini as entrypoint for proper signal handling +ENTRYPOINT ["/usr/bin/tini", "--"] + +# Default to running rfd-api, can be overridden with CMD +CMD ["rfd-api"] diff --git a/SETUP.md b/SETUP.md index 13a3119b..2416b351 100644 --- a/SETUP.md +++ b/SETUP.md @@ -107,3 +107,42 @@ cargo run -p rfd-cli --features local-dev The processor has multiple jobs that are able to be run, and configuration is only required for jobs that are going to be run. The `actions` key defines the jobs that should be run. By default all jobs are disabled. In this this mode the processor will only construct a database of RFDs. + +##### Static Asset Storage + +The processor can copy images and other static assets extracted from RFDs to cloud storage. Both +Google Cloud Storage (GCS) and Amazon S3 (or S3-compatible services) are supported. Multiple +storage backends can be configured simultaneously, and assets will be pushed to all of them. + +To enable this feature, add `CopyImagesToStorage` to the `actions` list and configure at least one +storage backend. + +**Google Cloud Storage (GCS)** + +```toml +[[gcs_storage]] +bucket = "your-bucket-name" +``` + +GCS uses GCP Application Default Credentials for authentication. Configure credentials using one of: +- `GOOGLE_APPLICATION_CREDENTIALS` environment variable pointing to a service account key file +- Instance metadata (when running on GCP Compute Engine, GKE, etc.) +- `gcloud auth application-default login` (for local development) + +**Amazon S3** + +```toml +[[s3_storage]] +bucket = "your-bucket-name" +region = "us-west-2" +# Optional: custom endpoint for S3-compatible services +# endpoint = "https://s3.example.com" +``` + +S3 uses the AWS SDK default credential chain for authentication: +- Environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, optionally `AWS_SESSION_TOKEN`) +- Shared credentials file (`~/.aws/credentials`) +- IAM role (when running on AWS EC2, ECS, Lambda, etc.) + +The optional `endpoint` field allows using S3-compatible services such as MinIO, Backblaze B2, or +Cloudflare R2. diff --git a/compose.yml b/compose.yml new file mode 100644 index 00000000..8378a8dc --- /dev/null +++ b/compose.yml @@ -0,0 +1,133 @@ +services: + # This requires changes to rfd-site which will be proposed via PR. + # rfd-site: + # build: + # context: https://github.com/oxidecomputer/rfd-site.git + # dockerfile: Containerfile + # image: localhost/rfd-site-test:latest + # networks: + # - rfd + # ports: + # - "3000:3000" + # environment: + # SESSION_SECRET: ${SESSION_SECRET} + # RFD_API_BACKEND_URL: http://rfd-api:8080 + # RFD_API_FRONTEND_URL: http://localhost:8080 + # RFD_API_CLIENT_ID: ${RFD_API_CLIENT_ID} + # RFD_API_CLIENT_SECRET: ${RFD_API_CLIENT_SECRET} + # RFD_API_GITHUB_CALLBACK_URL: http://localhost:3000/auth/github/callback + # AUTH_PROVIDERS: github + # STORAGE_PROVIDER: s3 + # S3_BUCKET: ${S3_BUCKET} + # AWS_REGION: ${AWS_REGION} + # AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + # AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + # AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN} + # TELEMETRY_DISABLE: true + # GITHUB_HOST: ${GITHUB_HOST} + # GITHUB_REPO_CLIENT_ID: ${GITHUB_REPO_CLIENT_ID} + # GITHUB_REPO_CLIENT_SECRET: ${GITHUB_REPO_CLIENT_SECRET} + + rfd-migration: + build: + dockerfile: Containerfile + image: localhost/rfd-api-test:latest + networks: + - rfd + depends_on: + postgres: + condition: service_healthy + entrypoint: [] + working_dir: /home/rfd/db + environment: + - DATABASE_URL=postgres://rfd:rfd@postgres/rfd + command: /usr/local/bin/rfd-installer && diesel migration run && echo "Migrations Complete!" + + rfd-api: + image: localhost/rfd-api-test:latest + # Uses default CMD: rfd-api + ports: + - "8080:8080" + environment: + # - RFD_API_CONFIG=/config/rfd-api.toml + - DATABASE_URL=postgres://rfd:rfd@postgres/rfd + command: + - /usr/local/bin/rfd-api + - /config/rfd-api.toml + volumes: + - ./config:/config:ro + networks: + - rfd + depends_on: + postgres: + condition: service_started + meilisearch: + condition: service_started + rfd-migration: + condition: service_completed_successfully + + rfd-processor: + image: localhost/rfd-api-test:latest + + # Override CMD to run rfd-processor instead + environment: + # - RFD_API_CONFIG=/config/rfd-api.toml + - DATABASE_URL=postgres://rfd:rfd@postgres/rfd + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + command: + - /usr/local/bin/rfd-processor + - /config/rfd-processor.toml + volumes: + - ./config:/config:ro + networks: + - rfd + depends_on: + postgres: + condition: service_started + meilisearch: + condition: service_started + rfd-migration: + condition: service_completed_successfully + + postgres: + image: docker.io/postgres:14-trixie + environment: + POSTGRES_USER: rfd + POSTGRES_PASSWORD: rfd + POSTGRES_DB: rfd + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - "5432:5432" + networks: + - rfd + healthcheck: + test: ["CMD-SHELL", "pg_isready -U rfd"] + start_period: 5s + start_interval: 1s + retries: 10 + interval: 5s + + meilisearch: + image: docker.io/getmeili/meilisearch:v1.11 + environment: + MEILI_ENV: development + MEILI_MASTER_KEY: development-master-key + MEILI_NO_ANALYTICS: "true" + MEILI_DB_PATH: /meili_data/data.ms + volumes: + - meilisearch_data:/meili_data + ports: + - "7700:7700" + networks: + - rfd + +networks: + rfd: + driver: bridge + +volumes: + postgres_data: + meilisearch_data: diff --git a/rfd-api/src/config.rs b/rfd-api/src/config.rs index 064ec64f..1179f41e 100644 --- a/rfd-api/src/config.rs +++ b/rfd-api/src/config.rs @@ -15,6 +15,7 @@ use crate::server::SpecConfig; #[derive(Debug, Deserialize)] pub struct AppConfig { pub log_format: ServerLogFormat, + pub log_filter: Option, pub log_directory: Option, pub initial_mappers: Option, pub public_url: String, diff --git a/rfd-api/src/main.rs b/rfd-api/src/main.rs index da4fd5f8..731934d4 100644 --- a/rfd-api/src/main.rs +++ b/rfd-api/src/main.rs @@ -55,10 +55,15 @@ async fn main() -> anyhow::Result<()> { NonBlocking::new(std::io::stdout()) }; + let env_filter = match config.log_filter { + Some(ref filter) => EnvFilter::new(filter), + None => EnvFilter::from_default_env(), + }; + let subscriber = tracing_subscriber::fmt() .with_file(false) .with_line_number(false) - .with_env_filter(EnvFilter::from_default_env()) + .with_env_filter(env_filter) .with_writer(writer); match config.log_format { diff --git a/rfd-api/src/server.rs b/rfd-api/src/server.rs index 70f19e04..61ae1f17 100644 --- a/rfd-api/src/server.rs +++ b/rfd-api/src/server.rs @@ -56,7 +56,7 @@ pub fn server( // Construct a shim to pipe dropshot logs into the global tracing logger let dropshot_logger = { - let level_drain = slog::LevelFilter(TracingSlogDrain, slog::Level::Debug).fuse(); + let level_drain = slog::LevelFilter(TracingSlogDrain, slog::Level::Trace).fuse(); let async_drain = slog_async::Async::new(level_drain).build().fuse(); slog::Logger::root(async_drain, slog::o!()) }; diff --git a/rfd-installer/src/lib.rs b/rfd-installer/src/lib.rs index e968fa6e..18c2a899 100644 --- a/rfd-installer/src/lib.rs +++ b/rfd-installer/src/lib.rs @@ -11,6 +11,7 @@ use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; const MIGRATIONS: EmbeddedMigrations = embed_migrations!("../rfd-model/migrations"); pub fn run_migrations(url: &str, v_only: bool) { + // These are safe to run multiple times. v_api_installer::run_migrations(url); if !v_only { diff --git a/rfd-model/diesel.toml b/rfd-model/diesel.toml index 2f2493e1..863549e3 100644 --- a/rfd-model/diesel.toml +++ b/rfd-model/diesel.toml @@ -1,9 +1,9 @@ # For documentation on how to configure this file, # see https://diesel.rs/guides/configuring-diesel-cli -[print_schema] -file = "src/schema.rs" -patch_file = "diesel-schema.patch" +# [print_schema] +# file = "src/schema.rs" +# patch_file = "diesel-schema.patch" [print_schema.filter] except_tables = [ diff --git a/rfd-processor/Cargo.toml b/rfd-processor/Cargo.toml index c853f700..178b9fb0 100644 --- a/rfd-processor/Cargo.toml +++ b/rfd-processor/Cargo.toml @@ -5,8 +5,13 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[dev-dependencies] +mockall = { workspace = true } + [dependencies] async-trait = { workspace = true } +aws-config = { workspace = true } +aws-sdk-s3 = { workspace = true } base64 = { workspace = true } chrono = { workspace = true } config = { workspace = true } diff --git a/rfd-processor/config.example.toml b/rfd-processor/config.example.toml index f33fbfed..385c7f38 100644 --- a/rfd-processor/config.example.toml +++ b/rfd-processor/config.example.toml @@ -69,10 +69,24 @@ path = "" # Branch to use as the default branch of the repository default_branch = "" -# Bucket to push static assets pulled from RFDs to (currently only GCP Storage buckets are supported) -[[static_storage]] -# Name of the bucket +# Static asset storage for images extracted from RFDs. Both GCS and S3 are supported. +# Multiple storage backends can be configured, and assets will be pushed to all of them. + +# Google Cloud Storage (GCS) +# Requires GCP Application Default Credentials to be configured +[[gcs_storage]] +# Name of the GCS bucket +bucket = "" + +# Amazon S3 (or S3-compatible storage) +# Uses AWS SDK default credential chain +[[s3_storage]] +# Name of the S3 bucket bucket = "" +# AWS region (e.g., "us-west-2") +region = "" +# Optional: Custom endpoint URL for S3-compatible services (e.g., MinIO, Backblaze B2) +# endpoint = "https://s3.example.com" # Location to store generated PDFs (currently on Google Drive Shared Drives are supported) [pdf_storage] diff --git a/rfd-processor/src/context.rs b/rfd-processor/src/context.rs index 7c5324b1..fbb6ce36 100644 --- a/rfd-processor/src/context.rs +++ b/rfd-processor/src/context.rs @@ -34,9 +34,22 @@ use crate::{ search::{RfdSearchIndex, SearchError}, updater::{BoxedAction, RfdUpdateMode, RfdUpdaterError}, util::{gdrive_client, GDriveError}, - AppConfig, GitHubAuthConfig, PdfStorageConfig, SearchConfig, StaticStorageConfig, + AppConfig, GcsStorageConfig, GitHubAuthConfig, PdfStorageConfig, S3StorageConfig, SearchConfig, }; +pub type StaticStorageError = Box; + +#[async_trait] +pub trait StaticStorage: Send + Sync { + async fn put( + &self, + key: &str, + data: Vec, + content_type: &str, + ) -> Result<(), StaticStorageError>; + fn name(&self) -> &str; +} + pub struct Database { pub storage: PostgresStore, } @@ -80,8 +93,8 @@ pub struct Context { pub db: Database, pub github: GitHubCtx, pub actions: Vec, - pub assets: StaticAssetStorageCtx, - pub pdf: PdfStorageCtx, + pub static_storage: Vec>, + pub pdf: Option, pub search: SearchCtx, } @@ -161,7 +174,7 @@ impl Context { .iter() .map(|action| action.as_str().try_into()) .collect::, RfdUpdaterError>>()?, - assets: StaticAssetStorageCtx::new(&config.static_storage).await?, + static_storage: build_static_storage(&config.gcs_storage, &config.s3_storage).await?, pdf: PdfStorageCtx::new(&config.pdf_storage).await?, search: SearchCtx::new(&config.search_storage)?, }) @@ -186,69 +199,150 @@ pub struct GitHubCtx { pub repository: GitHubRfdRepo, } -pub struct StaticAssetStorageCtx { - pub client: Storage>, - pub locations: Vec, +async fn build_static_storage( + gcs_entries: &[GcsStorageConfig], + s3_entries: &[S3StorageConfig], +) -> Result>, ContextError> { + let mut storage: Vec> = Vec::new(); + + // Build GCS storage instances + for entry in gcs_entries { + let client = build_gcs_client().await?; + storage.push(Box::new(GcsStorage { + client, + bucket: entry.bucket.clone(), + })); + } + + // Build S3 storage instances + for entry in s3_entries { + let mut config_loader = aws_config::defaults(aws_config::BehaviorVersion::latest()) + .region(aws_config::Region::new(entry.region.clone())); + + if let Some(endpoint) = &entry.endpoint { + config_loader = config_loader.endpoint_url(endpoint); + } + + let sdk_config = config_loader.load().await; + let client = aws_sdk_s3::Client::new(&sdk_config); + + storage.push(Box::new(S3Storage { + client, + bucket: entry.bucket.clone(), + })); + } + + Ok(storage) } -impl StaticAssetStorageCtx { - pub async fn new(entries: &[StaticStorageConfig]) -> Result { - let opts = yup_oauth2::ApplicationDefaultCredentialsFlowOpts::default(); - let gcp_auth = match yup_oauth2::ApplicationDefaultCredentialsAuthenticator::builder(opts) - .await - { - yup_oauth2::authenticator::ApplicationDefaultCredentialsTypes::ServiceAccount(auth) => { - tracing::debug!("Service account based credentials"); - - auth.build().await.map_err(|err| { - tracing::error!( - ?err, - "Failed to construct Cloud Storage credentials from service account" - ); - ContextError::FailedToFindGcpCredentials(err) - })? - } - yup_oauth2::authenticator::ApplicationDefaultCredentialsTypes::InstanceMetadata( - auth, - ) => { - tracing::debug!("Create instance based credentials"); - - auth.build().await.map_err(|err| { - tracing::error!( - ?err, - "Failed to construct Cloud Storage credentials from instance metadata" - ); - ContextError::FailedToFindGcpCredentials(err) - })? - } - }; +async fn build_gcs_client() -> Result>, ContextError> { + let opts = yup_oauth2::ApplicationDefaultCredentialsFlowOpts::default(); + let gcp_auth = match yup_oauth2::ApplicationDefaultCredentialsAuthenticator::builder(opts).await + { + yup_oauth2::authenticator::ApplicationDefaultCredentialsTypes::ServiceAccount(auth) => { + tracing::debug!("Service account based credentials"); + + auth.build().await.map_err(|err| { + tracing::error!( + ?err, + "Failed to construct Cloud Storage credentials from service account" + ); + ContextError::FailedToFindGcpCredentials(err) + })? + } + yup_oauth2::authenticator::ApplicationDefaultCredentialsTypes::InstanceMetadata(auth) => { + tracing::debug!("Create instance based credentials"); + + auth.build().await.map_err(|err| { + tracing::error!( + ?err, + "Failed to construct Cloud Storage credentials from instance metadata" + ); + ContextError::FailedToFindGcpCredentials(err) + })? + } + }; + + Ok(Storage::new( + Client::builder(TokioExecutor::new()).build( + HttpsConnectorBuilder::new() + .with_native_roots() + .unwrap() + .https_only() + .enable_http2() + .build(), + ), + gcp_auth, + )) +} - let storage = Storage::new( - Client::builder(TokioExecutor::new()).build( - HttpsConnectorBuilder::new() - .with_native_roots() - .unwrap() - .https_only() - .enable_http2() - .build(), - ), - gcp_auth, - ); +pub struct GcsStorage { + client: Storage>, + bucket: String, +} - Ok(Self { - client: storage, - locations: entries - .iter() - .map(|e| StaticAssetLocation { - bucket: e.bucket.to_string(), - }) - .collect(), - }) +#[async_trait] +impl StaticStorage for GcsStorage { + async fn put( + &self, + key: &str, + data: Vec, + content_type: &str, + ) -> Result<(), StaticStorageError> { + use google_storage1::api::Object; + + let mime_type: mime_guess::Mime = content_type + .parse() + .unwrap_or(mime_guess::mime::APPLICATION_OCTET_STREAM); + let cursor = std::io::Cursor::new(data); + + self.client + .objects() + .insert(Object::default(), &self.bucket) + .name(key) + .upload(cursor, mime_type) + .await?; + + Ok(()) + } + + fn name(&self) -> &str { + &self.bucket } } -pub struct StaticAssetLocation { - pub bucket: String, +pub struct S3Storage { + client: aws_sdk_s3::Client, + bucket: String, +} + +#[async_trait] +impl StaticStorage for S3Storage { + async fn put( + &self, + key: &str, + data: Vec, + content_type: &str, + ) -> Result<(), StaticStorageError> { + use aws_sdk_s3::primitives::ByteStream; + + let body = ByteStream::from(data); + + self.client + .put_object() + .bucket(&self.bucket) + .key(key) + .content_type(content_type) + .body(body) + .send() + .await?; + + Ok(()) + } + + fn name(&self) -> &str { + &self.bucket + } } pub type GDriveClient = DriveHub>; @@ -259,19 +353,16 @@ pub struct PdfStorageCtx { } impl PdfStorageCtx { - pub async fn new(config: &Option) -> Result { - Ok(Self { - // A client is only needed if files are going to be written - client: gdrive_client().await?, - locations: config - .as_ref() - .map(|config| { - vec![PdfStorageLocation { - folder_id: config.folder.clone(), - }] - }) - .unwrap_or_default(), - }) + pub async fn new(config: &Option) -> Result, GDriveError> { + match config { + Some(config) => Ok(Some(Self { + client: gdrive_client().await?, + locations: vec![PdfStorageLocation { + folder_id: config.folder.clone(), + }], + })), + None => Ok(None), + } } } @@ -324,7 +415,7 @@ impl PdfStorage for PdfStorageCtx { } } .tap_ok(|_| { - tracing::info!("Sucessfully uploaded PDF"); + tracing::info!("Successfully uploaded PDF"); }) .tap_err(|err| { tracing::error!(?err, "Failed to upload PDF"); @@ -364,3 +455,144 @@ impl SearchCtx { }) } } + +#[cfg(test)] +pub mod test { + use super::*; + use mockall::mock; + use std::collections::HashMap; + use std::sync::{Arc, Mutex}; + + mock! { + pub StaticStorage {} + + #[async_trait] + impl StaticStorage for StaticStorage { + async fn put( + &self, + key: &str, + data: Vec, + content_type: &str, + ) -> Result<(), StaticStorageError>; + fn name(&self) -> &str; + } + } + + /// In-memory storage implementation for testing + pub struct InMemoryStorage { + name: String, + objects: Arc>>, + } + + #[derive(Clone)] + pub struct StoredObject { + pub data: Vec, + pub content_type: String, + } + + impl InMemoryStorage { + pub fn new(name: &str) -> Self { + Self { + name: name.to_string(), + objects: Arc::new(Mutex::new(HashMap::new())), + } + } + + pub fn get(&self, key: &str) -> Option { + self.objects.lock().unwrap().get(key).cloned() + } + + pub fn len(&self) -> usize { + self.objects.lock().unwrap().len() + } + } + + #[async_trait] + impl StaticStorage for InMemoryStorage { + async fn put( + &self, + key: &str, + data: Vec, + content_type: &str, + ) -> Result<(), StaticStorageError> { + self.objects.lock().unwrap().insert( + key.to_string(), + StoredObject { + data, + content_type: content_type.to_string(), + }, + ); + Ok(()) + } + + fn name(&self) -> &str { + &self.name + } + } + + #[tokio::test] + async fn in_memory_storage_stores_and_retrieves_objects() { + let storage = InMemoryStorage::new("test-bucket"); + + let data = b"hello world".to_vec(); + storage + .put("test/key.txt", data.clone(), "text/plain") + .await + .unwrap(); + + let obj = storage.get("test/key.txt").unwrap(); + assert_eq!(obj.data, data); + assert_eq!(obj.content_type, "text/plain"); + } + + #[tokio::test] + async fn in_memory_storage_returns_correct_name() { + let storage = InMemoryStorage::new("my-bucket"); + assert_eq!(storage.name(), "my-bucket"); + } + + #[tokio::test] + async fn in_memory_storage_overwrites_existing_objects() { + let storage = InMemoryStorage::new("test-bucket"); + + storage + .put("key", b"first".to_vec(), "text/plain") + .await + .unwrap(); + storage + .put("key", b"second".to_vec(), "text/plain") + .await + .unwrap(); + + let obj = storage.get("key").unwrap(); + assert_eq!(obj.data, b"second".to_vec()); + assert_eq!(storage.len(), 1); + } + + #[tokio::test] + async fn mock_storage_can_simulate_failure() { + let mut mock = MockStaticStorage::new(); + mock.expect_put() + .returning(|_, _, _| Err("simulated failure".into())); + mock.expect_name().return_const("mock-bucket".to_string()); + + let result = mock.put("key", vec![], "text/plain").await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn mock_storage_can_verify_calls() { + let mut mock = MockStaticStorage::new(); + mock.expect_put() + .withf(|key, data, content_type| { + key == "expected/key.png" && data == b"image data" && content_type == "image/png" + }) + .times(1) + .returning(|_, _, _| Ok(())); + mock.expect_name().return_const("mock-bucket".to_string()); + + mock.put("expected/key.png", b"image data".to_vec(), "image/png") + .await + .unwrap(); + } +} diff --git a/rfd-processor/src/main.rs b/rfd-processor/src/main.rs index 0641d2b5..895ff0ab 100644 --- a/rfd-processor/src/main.rs +++ b/rfd-processor/src/main.rs @@ -31,6 +31,7 @@ mod util; #[derive(Debug, Deserialize)] pub struct AppConfig { pub log_directory: Option, + pub log_filter: Option, #[serde(default)] pub log_format: LogFormat, pub processor_enabled: bool, @@ -45,7 +46,9 @@ pub struct AppConfig { pub auth: AuthConfig, pub source: GitHubSourceRepo, #[serde(default)] - pub static_storage: Vec, + pub gcs_storage: Vec, + #[serde(default)] + pub s3_storage: Vec, #[serde(default)] pub pdf_storage: Option, #[serde(default)] @@ -96,10 +99,17 @@ pub struct GitHubSourceRepo { } #[derive(Debug, Deserialize, Serialize)] -pub struct StaticStorageConfig { +pub struct GcsStorageConfig { pub bucket: String, } +#[derive(Debug, Deserialize, Serialize)] +pub struct S3StorageConfig { + pub bucket: String, + pub region: String, + pub endpoint: Option, +} + #[derive(Debug, Deserialize, Serialize)] pub struct PdfStorageConfig { pub drive: Option, @@ -145,10 +155,15 @@ async fn main() -> Result<(), Box> { NonBlocking::new(std::io::stdout()) }; + let env_filter = match config.log_filter { + Some(ref filter) => EnvFilter::new(filter), + None => EnvFilter::from_default_env(), + }; + let subscriber = tracing_subscriber::fmt() .with_file(false) .with_line_number(false) - .with_env_filter(EnvFilter::from_default_env()) + .with_env_filter(env_filter) .with_writer(writer); match config.log_format { diff --git a/rfd-processor/src/updater/copy_images_to_storage.rs b/rfd-processor/src/updater/copy_images_to_storage.rs index ee72a669..73a7768f 100644 --- a/rfd-processor/src/updater/copy_images_to_storage.rs +++ b/rfd-processor/src/updater/copy_images_to_storage.rs @@ -3,7 +3,6 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use async_trait::async_trait; -use google_storage1::api::Object; use tracing::instrument; use crate::{rfd::PersistedRfd, util::decode_base64}; @@ -49,24 +48,25 @@ impl RfdUpdateAction for CopyImagesToStorage { "Writing file to storage buckets" ); - let cursor = std::io::Cursor::new(data); - - for location in &ctx.assets.locations { - tracing::info!(bucket = ?location.bucket, ?object_name, "Writing to location"); + for storage in &ctx.static_storage { + tracing::info!(name = storage.name(), ?object_name, "Writing to storage"); if mode == RfdUpdateMode::Write { - // TODO: Move implementation to a trait and abstract over different storage systems - if let Err(err) = ctx - .assets - .client - .objects() - .insert(Object::default(), &location.bucket) - .name(&object_name) - .upload(cursor.clone(), mime_type.clone()) + if let Err(err) = storage + .put(&object_name, data.clone(), mime_type.as_ref()) .await { - tracing::error!(?err, "Failed to upload static file to GCP"); + tracing::error!( + name = storage.name(), + ?err, + "Failed to upload static file" + ); } + } else { + tracing::warn!( + "CopyImagesToStorage is enabled however RfdUpdateMode is not write: {:?}", + mode + ); } } } diff --git a/rfd-processor/src/updater/update_pdfs.rs b/rfd-processor/src/updater/update_pdfs.rs index 365dcac0..1c20d992 100644 --- a/rfd-processor/src/updater/update_pdfs.rs +++ b/rfd-processor/src/updater/update_pdfs.rs @@ -68,15 +68,21 @@ impl UpdatePdfs { let store_results = match mode { RfdUpdateMode::Read => Vec::new(), - RfdUpdateMode::Write => { - ctx.pdf - .store_rfd_pdf( - new.pdf_external_id.as_deref(), - &new.get_pdf_filename(), - &pdf, - ) - .await - } + RfdUpdateMode::Write => match &ctx.pdf { + Some(pdf_storage) => { + pdf_storage + .store_rfd_pdf( + new.pdf_external_id.as_deref(), + &new.get_pdf_filename(), + &pdf, + ) + .await + } + None => { + tracing::debug!("PDF storage is disabled, skipping upload"); + Vec::new() + } + }, }; Ok(store_results