From 74c13fb055d66e416439614dda035e48425727af Mon Sep 17 00:00:00 2001 From: Nick Downs Date: Tue, 3 Feb 2026 16:02:34 -0800 Subject: [PATCH 1/2] Container support and S3 support for static storage * Added Containerfile and compose.yml allowing for rfd-api and all other necessary binaries to be built into a container image. The compose.yml can be used to stand-up a local stack which includes the database and meilisearch. * Added log_filter support to both the rfd-api and rfd-processor configuration files. This works the same as overriding RUST_LOG but allows for configuration via file instead of environment variable. * StaticStorage is now a trait. There are two implementations for the trait. Each implementation have corresponding configuration sections. Multiple static storage locations can be configured and processor will upload static storage to all configured static storage backends. ** GCS This is the original GCS implementation. ** S3 This is a new implementation using the S3 API. * PDF storage is now optional. Disable this feature by not configuring it in the configuration toml. --- .gitignore | 2 +- Cargo.lock | 1091 +++++++++++++++-- Cargo.toml | 2 + Containerfile | 78 ++ SETUP.md | 39 + compose.yml | 131 ++ rfd-api/src/config.rs | 1 + rfd-api/src/main.rs | 7 +- rfd-api/src/server.rs | 2 +- rfd-installer/src/lib.rs | 1 + rfd-model/diesel.toml | 6 +- rfd-processor/Cargo.toml | 5 + rfd-processor/config.example.toml | 20 +- rfd-processor/src/context.rs | 380 ++++-- rfd-processor/src/main.rs | 21 +- .../src/updater/copy_images_to_storage.rs | 28 +- rfd-processor/src/updater/update_pdfs.rs | 24 +- 17 files changed, 1607 insertions(+), 231 deletions(-) create mode 100644 Containerfile create mode 100644 compose.yml diff --git a/.gitignore b/.gitignore index 0704a0eb..7156a1fc 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,4 @@ node_modules .rustup /config/ local_data -.env \ No newline at end of file +.env diff --git a/Cargo.lock b/Cargo.lock index c6a6af89..ba3cc583 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,7 +91,7 @@ checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" [[package]] name = "async-bb8-diesel" version = "0.3.0" -source = "git+https://github.com/oxidecomputer/async-bb8-diesel#18baf49aca5f72bf441951b11669c6c3c3affe70" +source = "git+https://github.com/oxidecomputer/async-bb8-diesel#f049570ff89080b2385c637af4c0e614b0a124a4" dependencies = [ "async-trait", "bb8", @@ -157,6 +157,453 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-config" +version = "1.8.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c456581cb3c77fafcc8c67204a70680d40b61112d6da78c77bd31d945b65f1b5" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 1.4.0", + "ring", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd362783681b15d136480ad555a099e82ecd8e2d10a841e14dfd0078d67fee3" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-lc-rs" +version = "1.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b7b6141e96a8c160799cc2d5adecd5cbbe5054cb8c7c4af53da0f83bb7ad256" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c34dda4df7017c8db52132f0f8a2e0f8161649d15723ed63fc00c82d0f2081a" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "aws-runtime" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c635c2dc792cb4a11ce1a4f392a925340d1bdf499289b5ec1ec6810954eb43f5" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.122.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94c2ca0cba97e8e279eb6c0b2d0aa10db5959000e602ab2b7c02de6b85d4c19b" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.0", + "http-body 1.0.1", + "lru", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.93.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dcb38bb33fc0a11f1ffc3e3e85669e0a11a37690b86f77e75306d8f369146a0" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.95.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ada8ffbea7bd1be1f53df1dadb0f8fdb04badb13185b3321b929d1ee3caad09" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.97.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6443ccadc777095d5ed13e21f5c364878c9f5bad4e35187a6cdbd863b0afcad" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efa49f3c607b92daae0c078d48a4571f599f966dce3caee5f1ea55c4d9073f99" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.0", + "p256 0.11.1", + "percent-encoding", + "ring", + "sha2", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52eec3db979d18cb807fc1070961cc51d87d069abe9ab57917769687368a8c6c" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.64.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcf418858f9f3edd228acb8759d77394fed7531cce78d02bdda499025368439" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc-fast", + "hex", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35b9c7354a3b13c66f60fe4616d6d1969c9fd36b1b5333a5dfb3ee716b33c588" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.63.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630e67f2a31094ffa51b210ae030855cb8f3b7ee1329bdd8d085aaf61e8b97fc" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12fb0abf49ff0cab20fd31ac1215ed7ce0ea92286ba09e2854b42ba5cabe7525" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.3.27", + "h2 0.4.13", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper 1.8.1", + "hyper-rustls 0.24.2", + "hyper-rustls 0.27.7", + "hyper-util", + "pin-project-lite", + "rustls 0.21.12", + "rustls 0.23.36", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.4", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.62.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cb96aa208d62ee94104645f7b2ecaf77bf27edf161590b6224bfbac2832f979" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0a46543fbc94621080b3cf553eb4cbbdc41dd9780a30c4756400f0139440a1d" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cebbddb6f3a5bd81553643e9c7daf3cc3dc5b0b5f398ac668630e8a84e6fff0" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3df87c14f0127a0d77eb261c3bc45d5b4833e2a1f63583ebfb728e4852134ee" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49952c52f7eebb72ce2a754d3866cc0f87b97d2a46146b79f80f3a93fb2b3716" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.4.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3a26048eeab0ddeba4b4f9d51654c79af8c3b32357dc5f336cee85ab331c33" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11b2f670422ff42bf7065031e72b45bc52a3508bd089f743ea90731ca2b6ea57" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d980627d2dd7bfc32a3c025685a033eeab8d365cc840c631ef59d1b8f428164" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base16ct" version = "0.2.0" @@ -169,6 +616,16 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.8.3" @@ -225,13 +682,23 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" dependencies = [ "serde", ] +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "camino" version = "1.2.2" @@ -243,11 +710,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.54" +version = "1.2.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6354c81bbfd62d9cfa9cb3c773c2b7b2a3a482d569de977fd0e961f6e7c00583" +checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -279,9 +748,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.54" +version = "4.5.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394" +checksum = "6899ea499e3fb9305a65d5ebf6e3d2248c5fab291f300ad0a704fbe142eae31a" dependencies = [ "clap_builder", "clap_derive", @@ -289,9 +758,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.54" +version = "4.5.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00" +checksum = "7b12c8b680195a62a8364d16b8447b01b6c2c8f9aaf68bee653be34d4245e238" dependencies = [ "anstream", "anstyle", @@ -301,9 +770,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.49" +version = "4.5.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" dependencies = [ "heck", "proc-macro2", @@ -317,6 +786,15 @@ version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32" +[[package]] +name = "cmake" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.4" @@ -423,6 +901,33 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc-fast" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd92aca2c6001b1bf5ba0ff84ee74ec8501b52bbef0cac80bf25a6c1d87a83d" +dependencies = [ + "crc", + "digest", + "rustversion", + "spin 0.10.0", +] + [[package]] name = "crc32c" version = "0.6.8" @@ -432,6 +937,15 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -451,7 +965,19 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" name = "crunchy" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] [[package]] name = "crypto-bigint" @@ -557,6 +1083,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffe7ed1d93f4553003e20b629abe9085e1e81b1429520f897f8f8860bc6dfc21" +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der" version = "0.7.10" @@ -716,9 +1252,9 @@ checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" [[package]] name = "dropshot" -version = "0.16.6" +version = "0.16.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d0df98c06659ab85a454f32dc36ca5dbc6500bd2a58f25ede4dc1f1d478904e" +checksum = "d69fd85c8dfc67252d02f260595f6b62b5abceb1b88b4b9722369d27936e5fa4" dependencies = [ "async-stream", "async-trait", @@ -733,7 +1269,7 @@ dependencies = [ "hostname 0.4.2", "http 1.4.0", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-util", "indexmap 2.13.0", "multer", @@ -767,7 +1303,7 @@ dependencies = [ [[package]] name = "dropshot-authorization-header" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/v-api#5ea039de839c4a88943961e8bbfb298781671755" +source = "git+https://github.com/oxidecomputer/v-api#21d760b210429e23df76a1f6bf691c94b3de5488" dependencies = [ "async-trait", "base64", @@ -786,7 +1322,7 @@ dependencies = [ "hex", "hmac", "http 1.4.0", - "hyper", + "hyper 1.8.1", "schemars 0.8.22", "serde", "serde_json", @@ -797,9 +1333,9 @@ dependencies = [ [[package]] name = "dropshot_endpoint" -version = "0.16.6" +version = "0.16.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e53aef8838e0e341485590738ab180a6dceff3565ffcb198d5f365fea650378" +checksum = "67d106478e4a4782556981d028a667f41c4845cdaa6e2d3a9f58c5d15e725401" dependencies = [ "heck", "proc-macro2", @@ -824,24 +1360,42 @@ dependencies = [ "syn", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der 0.6.1", + "elliptic-curve 0.12.3", + "rfc6979 0.3.1", + "signature 1.6.4", +] + [[package]] name = "ecdsa" version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der", + "der 0.7.10", "digest", - "elliptic-curve", - "rfc6979", - "signature", - "spki", + "elliptic-curve 0.13.8", + "rfc6979 0.4.0", + "signature 2.2.0", + "spki 0.7.3", ] [[package]] @@ -859,8 +1413,8 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", - "signature", + "pkcs8 0.10.2", + "signature 2.2.0", ] [[package]] @@ -886,23 +1440,43 @@ dependencies = [ "serde", ] +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct 0.1.1", + "crypto-bigint 0.4.9", + "der 0.6.1", + "digest", + "ff 0.12.1", + "generic-array", + "group 0.12.1", + "pkcs8 0.9.0", + "rand_core 0.6.4", + "sec1 0.3.0", + "subtle", + "zeroize", +] + [[package]] name = "elliptic-curve" version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ - "base16ct", - "crypto-bigint", + "base16ct 0.2.0", + "crypto-bigint 0.5.5", "digest", - "ff", + "ff 0.13.1", "generic-array", - "group", + "group 0.13.0", "hkdf", "pem-rfc7468", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "sec1", + "sec1 0.7.3", "subtle", "zeroize", ] @@ -958,6 +1532,16 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "ff" version = "0.13.1" @@ -976,9 +1560,9 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "find-msvc-tools" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "fnv" @@ -1013,6 +1597,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619" +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.31" @@ -1156,7 +1746,7 @@ dependencies = [ "chrono", "http 1.4.0", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-util", "mime", "percent-encoding", @@ -1176,8 +1766,8 @@ checksum = "cef55117f2b1d40e56f2fd26161a2e93c9d7899be9389a9d4aa59308168e880c" dependencies = [ "chrono", "google-apis-common", - "hyper", - "hyper-rustls", + "hyper 1.8.1", + "hyper-rustls 0.27.7", "hyper-util", "mime", "serde", @@ -1196,8 +1786,8 @@ checksum = "30742c5730cad187ffb8922f1a91766830676ec6b714489fcdb89e9b9385fd7c" dependencies = [ "chrono", "google-apis-common", - "hyper", - "hyper-rustls", + "hyper 1.8.1", + "hyper-rustls 0.27.7", "hyper-util", "mime", "serde", @@ -1216,8 +1806,8 @@ checksum = "0d454a68994bbdae724ec6a3129a23ec2a86ef29f8ae9e92bb3a3578c193284c" dependencies = [ "chrono", "google-apis-common", - "hyper", - "hyper-rustls", + "hyper 1.8.1", + "hyper-rustls 0.27.7", "hyper-util", "mime", "serde", @@ -1251,17 +1841,47 @@ dependencies = [ "web-time", ] +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff 0.12.1", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "group" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ - "ff", + "ff 0.13.1", "rand_core 0.6.4", "subtle", ] +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.13.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.13" @@ -1410,6 +2030,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -1429,7 +2060,7 @@ dependencies = [ "bytes", "futures-core", "http 1.4.0", - "http-body", + "http-body 1.0.1", "pin-project-lite", ] @@ -1445,6 +2076,30 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.8.1" @@ -1455,9 +2110,9 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", + "h2 0.4.13", "http 1.4.0", - "http-body", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -1468,6 +2123,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.7" @@ -1475,7 +2145,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ "http 1.4.0", - "hyper", + "hyper 1.8.1", "hyper-util", "rustls 0.23.36", "rustls-native-certs", @@ -1488,23 +2158,22 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ "base64", "bytes", "futures-channel", - "futures-core", "futures-util", "http 1.4.0", - "http-body", - "hyper", + "http-body 1.0.1", + "hyper 1.8.1", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.2", "system-configuration", "tokio", "tower-layer", @@ -1515,9 +2184,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.64" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1737,6 +2406,16 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.85" @@ -1775,16 +2454,16 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "10.2.0" +version = "10.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c76e1c7d7df3e34443b3621b459b066a7b79644f059fc8b2db7070c825fd417e" +checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1" dependencies = [ "base64", "ed25519-dalek", "getrandom 0.2.17", "hmac", "js-sys", - "p256", + "p256 0.13.2", "p384", "pem", "rand 0.8.5", @@ -1792,7 +2471,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "signature", + "signature 2.2.0", "simple_asn1", ] @@ -1802,7 +2481,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin", + "spin 0.9.8", ] [[package]] @@ -1857,6 +2536,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "lru" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -1996,9 +2684,9 @@ dependencies = [ [[package]] name = "minijinja" -version = "2.14.0" +version = "2.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ea9ac0a51fb5112607099560fdf0f90366ab088a2a9e6e8ae176794e9806aa" +checksum = "b479616bb6f0779fb0f3964246beda02d4b01144e1b0d5519616e012ccc2a245" dependencies = [ "memo-map", "self_cell", @@ -2055,7 +2743,7 @@ dependencies = [ "httparse", "memchr", "mime", - "spin", + "spin 0.9.8", "version_check", ] @@ -2274,20 +2962,37 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "owo-colors" version = "4.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c6901729fa79e91a0913333229e9ca5dc725089d1c363b2f4b4760709dc4a52" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa 0.14.8", + "elliptic-curve 0.12.3", + "sha2", +] + [[package]] name = "p256" version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ - "ecdsa", - "elliptic-curve", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", "primeorder", "sha2", ] @@ -2298,8 +3003,8 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" dependencies = [ - "ecdsa", - "elliptic-curve", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", "primeorder", "sha2", ] @@ -2480,9 +3185,19 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", - "pkcs8", - "spki", + "der 0.7.10", + "pkcs8 0.10.2", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der 0.6.1", + "spki 0.6.0", ] [[package]] @@ -2491,8 +3206,8 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", ] [[package]] @@ -2503,9 +3218,9 @@ checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "portable-atomic" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "potential_utf" @@ -2574,7 +3289,7 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" dependencies = [ - "elliptic-curve", + "elliptic-curve 0.13.8", ] [[package]] @@ -2680,7 +3395,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.23.36", - "socket2", + "socket2 0.6.2", "thiserror 2.0.18", "tokio", "tracing", @@ -2717,7 +3432,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.6.2", "tracing", "windows-sys 0.60.2", ] @@ -2878,9 +3593,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.2" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", @@ -2890,20 +3605,26 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" [[package]] name = "regress" @@ -2925,12 +3646,12 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "h2", + "h2 0.4.13", "http 1.4.0", - "http-body", + "http-body 1.0.1", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.8.1", + "hyper-rustls 0.27.7", "hyper-util", "js-sys", "log", @@ -2996,7 +3717,7 @@ dependencies = [ "futures", "getrandom 0.2.17", "http 1.4.0", - "hyper", + "hyper 1.8.1", "parking_lot 0.11.2", "reqwest", "reqwest-middleware", @@ -3049,6 +3770,17 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -3078,8 +3810,8 @@ dependencies = [ "dropshot-verified-body", "hex", "http 1.4.0", - "hyper", - "jsonwebtoken 10.2.0", + "hyper 1.8.1", + "jsonwebtoken 10.3.0", "meilisearch-sdk", "minijinja", "mockall", @@ -3136,7 +3868,7 @@ dependencies = [ "dirs 6.0.0", "futures", "itertools", - "jsonwebtoken 10.2.0", + "jsonwebtoken 10.3.0", "oauth2", "owo-colors", "progenitor-client", @@ -3217,6 +3949,8 @@ name = "rfd-processor" version = "0.12.3" dependencies = [ "async-trait", + "aws-config", + "aws-sdk-s3", "base64", "chrono", "config", @@ -3230,6 +3964,7 @@ dependencies = [ "md-5", "meilisearch-sdk", "mime_guess", + "mockall", "newtype-uuid", "octorust", "parse-rfd", @@ -3307,11 +4042,11 @@ dependencies = [ "num-integer", "num-traits", "pkcs1", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", "sha2", - "signature", - "spki", + "signature 2.2.0", + "spki 0.7.3", "subtle", "zeroize", ] @@ -3367,6 +4102,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.22.4" @@ -3387,6 +4134,7 @@ version = "0.23.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" dependencies = [ + "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", @@ -3426,6 +4174,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.102.8" @@ -3443,6 +4201,7 @@ version = "0.103.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -3517,9 +4276,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54e910108742c57a770f492731f99be216a52fadd361b06c8fb59d74ccc267d2" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" dependencies = [ "dyn-clone", "ref-cast", @@ -3545,22 +4304,46 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "seahash" version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct 0.1.1", + "der 0.6.1", + "generic-array", + "pkcs8 0.9.0", + "subtle", + "zeroize", +] + [[package]] name = "sec1" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "base16ct", - "der", + "base16ct 0.2.0", + "der 0.7.10", "generic-array", - "pkcs8", + "pkcs8 0.10.2", "subtle", "zeroize", ] @@ -3745,7 +4528,7 @@ dependencies = [ "indexmap 1.9.3", "indexmap 2.13.0", "schemars 0.9.0", - "schemars 1.2.0", + "schemars 1.2.1", "serde_core", "serde_json", "serde_with_macros", @@ -3824,6 +4607,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "signature" version = "2.2.0" @@ -3854,9 +4647,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "slog" @@ -3926,6 +4719,16 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.2" @@ -3942,6 +4745,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + [[package]] name = "spinning_top" version = "0.3.0" @@ -3951,6 +4760,16 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der 0.6.1", +] + [[package]] name = "spki" version = "0.7.3" @@ -3958,7 +4777,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", ] [[package]] @@ -4035,9 +4854,9 @@ dependencies = [ [[package]] name = "system-configuration" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ "bitflags 2.10.0", "core-foundation 0.9.4", @@ -4231,7 +5050,7 @@ dependencies = [ "parking_lot 0.12.5", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.6.2", "tokio-macros", "windows-sys 0.61.2", ] @@ -4247,6 +5066,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.25.0" @@ -4399,7 +5228,7 @@ dependencies = [ "bytes", "futures-util", "http 1.4.0", - "http-body", + "http-body 1.0.1", "iri-string", "pin-project-lite", "tower", @@ -4646,6 +5475,12 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -4673,7 +5508,7 @@ dependencies = [ [[package]] name = "v-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/v-api#5ea039de839c4a88943961e8bbfb298781671755" +source = "git+https://github.com/oxidecomputer/v-api#21d760b210429e23df76a1f6bf691c94b3de5488" dependencies = [ "async-trait", "base64", @@ -4688,8 +5523,8 @@ dependencies = [ "hex", "http 1.4.0", "http-body-util", - "hyper", - "jsonwebtoken 10.2.0", + "hyper 1.8.1", + "jsonwebtoken 10.3.0", "newtype-uuid", "oauth2", "partial-struct", @@ -4718,7 +5553,7 @@ dependencies = [ [[package]] name = "v-api-installer" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/v-api#5ea039de839c4a88943961e8bbfb298781671755" +source = "git+https://github.com/oxidecomputer/v-api#21d760b210429e23df76a1f6bf691c94b3de5488" dependencies = [ "diesel", "diesel_migrations", @@ -4727,7 +5562,7 @@ dependencies = [ [[package]] name = "v-api-permission-derive" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/v-api#5ea039de839c4a88943961e8bbfb298781671755" +source = "git+https://github.com/oxidecomputer/v-api#21d760b210429e23df76a1f6bf691c94b3de5488" dependencies = [ "heck", "proc-macro2", @@ -4738,7 +5573,7 @@ dependencies = [ [[package]] name = "v-model" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/v-api#5ea039de839c4a88943961e8bbfb298781671755" +source = "git+https://github.com/oxidecomputer/v-api#21d760b210429e23df76a1f6bf691c94b3de5488" dependencies = [ "async-bb8-diesel", "async-trait", @@ -4773,6 +5608,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "waitgroup" version = "0.1.2" @@ -4925,9 +5766,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" dependencies = [ "rustls-pki-types", ] @@ -5210,6 +6051,12 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "xtask" version = "0.0.0" @@ -5280,8 +6127,8 @@ dependencies = [ "base64", "http 1.4.0", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.8.1", + "hyper-rustls 0.27.7", "hyper-util", "log", "percent-encoding", @@ -5297,18 +6144,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.34" +version = "0.8.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71ddd76bcebeed25db614f82bf31a9f4222d3fbba300e6fb6c00afa26cbd4d9d" +checksum = "57cf3aa6855b23711ee9852dfc97dfaa51c45feaba5b645d0c777414d494a961" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.34" +version = "0.8.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8187381b52e32220d50b255276aa16a084ec0a9017a0ca2152a1f55c539758d" +checksum = "8a616990af1a287837c4fe6596ad77ef57948f787e46ce28e166facc0cc1cb75" dependencies = [ "proc-macro2", "quote", @@ -5377,6 +6224,6 @@ dependencies = [ [[package]] name = "zmij" -version = "1.0.17" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02aae0f83f69aafc94776e879363e9771d7ecbffe2c7fbb6c14c5e00dfe88439" +checksum = "3ff05f8caa9038894637571ae6b9e29466c1f4f829d26c9b28f869a29cbe3445" diff --git a/Cargo.toml b/Cargo.toml index b74c6c61..538a7872 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,8 @@ resolver = "2" anyhow = "1.0.100" async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", version = "0.3" } async-trait = "0.1.89" +aws-config = "1.8.13" +aws-sdk-s3 = "1.99.0" base64 = "0.22" bb8 = "0.9" chrono = "0.4.43" diff --git a/Containerfile b/Containerfile new file mode 100644 index 00000000..3c0dcfaf --- /dev/null +++ b/Containerfile @@ -0,0 +1,78 @@ +# syntax=docker/dockerfile:1 +# Build stage +FROM docker.io/rust:1-trixie AS builder + +ARG DIESEL_VER=v2.3.5 + +WORKDIR /app + +# Install build dependencies for diesel/postgres +RUN apt-get update && apt-get install -y \ + libpq-dev \ + pkg-config + + +# Copy workspace files +COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ + +# Copy all workspace members +COPY parse-rfd ./parse-rfd +COPY rfd-api ./rfd-api +COPY rfd-cli ./rfd-cli +COPY rfd-data ./rfd-data +COPY rfd-github ./rfd-github +COPY rfd-installer ./rfd-installer +COPY rfd-model ./rfd-model +COPY rfd-processor ./rfd-processor +COPY rfd-sdk ./rfd-sdk +COPY trace-request ./trace-request +COPY xtask ./xtask + +ENV CARGO_HOME=/data/cargo + +# Build all target binaries in release mode +RUN cargo build --release \ + --package rfd-api \ + --package rfd-processor \ + --package rfd-cli \ + --package rfd-installer + +# Download diesel tool for migrations +WORKDIR /tmp +RUN curl -L --output-dir /tmp -O "https://github.com/diesel-rs/diesel/releases/download/${DIESEL_VER}/diesel_cli-x86_64-unknown-linux-gnu.tar.xz" \ + && curl -L --output-dir /tmp -O "https://github.com/diesel-rs/diesel/releases/download/${DIESEL_VER}/diesel_cli-x86_64-unknown-linux-gnu.tar.xz.sha256" \ + && sha256sum diesel_cli-x86_64-unknown-linux-gnu.tar.xz.sha256 && tar --strip-components=1 -xJvf diesel_cli-x86_64-unknown-linux-gnu.tar.xz + +# Runtime stage +FROM docker.io/debian:trixie-slim + +# Install runtime dependencies and tini +RUN apt-get update && apt-get install -y \ + ca-certificates \ + libpq5 \ + tini \ + && rm -rf /var/lib/apt/lists/* + +RUN useradd --create-home --user-group rfd \ + && mkdir /home/rfd/db + +# Copy binaries from builder +COPY --from=builder /app/target/release/rfd-api /usr/local/bin/ +COPY --from=builder /app/target/release/rfd-processor /usr/local/bin/ +COPY --from=builder /app/target/release/rfd-cli /usr/local/bin/ +COPY --from=builder /app/target/release/rfd-installer /usr/local/bin/ + +# Database migrations for diesel +COPY --from=builder /tmp/diesel /usr/local/bin/ +# COPY --from=builder /app/rfd-model/diesel.toml /home/rfd/db/ +COPY --from=builder /app/rfd-model/migrations/ /home/rfd/db/migrations/ + +# Create non-root user +USER rfd +WORKDIR /home/rfd + +# Use tini as entrypoint for proper signal handling +ENTRYPOINT ["/usr/bin/tini", "--"] + +# Default to running rfd-api, can be overridden with CMD +CMD ["rfd-api"] diff --git a/SETUP.md b/SETUP.md index 13a3119b..2416b351 100644 --- a/SETUP.md +++ b/SETUP.md @@ -107,3 +107,42 @@ cargo run -p rfd-cli --features local-dev The processor has multiple jobs that are able to be run, and configuration is only required for jobs that are going to be run. The `actions` key defines the jobs that should be run. By default all jobs are disabled. In this this mode the processor will only construct a database of RFDs. + +##### Static Asset Storage + +The processor can copy images and other static assets extracted from RFDs to cloud storage. Both +Google Cloud Storage (GCS) and Amazon S3 (or S3-compatible services) are supported. Multiple +storage backends can be configured simultaneously, and assets will be pushed to all of them. + +To enable this feature, add `CopyImagesToStorage` to the `actions` list and configure at least one +storage backend. + +**Google Cloud Storage (GCS)** + +```toml +[[gcs_storage]] +bucket = "your-bucket-name" +``` + +GCS uses GCP Application Default Credentials for authentication. Configure credentials using one of: +- `GOOGLE_APPLICATION_CREDENTIALS` environment variable pointing to a service account key file +- Instance metadata (when running on GCP Compute Engine, GKE, etc.) +- `gcloud auth application-default login` (for local development) + +**Amazon S3** + +```toml +[[s3_storage]] +bucket = "your-bucket-name" +region = "us-west-2" +# Optional: custom endpoint for S3-compatible services +# endpoint = "https://s3.example.com" +``` + +S3 uses the AWS SDK default credential chain for authentication: +- Environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, optionally `AWS_SESSION_TOKEN`) +- Shared credentials file (`~/.aws/credentials`) +- IAM role (when running on AWS EC2, ECS, Lambda, etc.) + +The optional `endpoint` field allows using S3-compatible services such as MinIO, Backblaze B2, or +Cloudflare R2. diff --git a/compose.yml b/compose.yml new file mode 100644 index 00000000..136ff31c --- /dev/null +++ b/compose.yml @@ -0,0 +1,131 @@ +services: + # This requires changes to rfd-site which will be proposed via PR. + # rfd-site: + # build: + # context: https://github.com/oxidecomputer/rfd-site.git + # dockerfile: Containerfile + # image: localhost/rfd-site-test:latest + # networks: + # - rfd + # ports: + # - "3000:3000" + # environment: + # SESSION_SECRET: ${SESSION_SECRET} + # RFD_API_BACKEND_URL: http://rfd-api:8080 + # RFD_API_FRONTEND_URL: http://localhost:8080 + # RFD_API_CLIENT_ID: ${RFD_API_CLIENT_ID} + # RFD_API_CLIENT_SECRET: ${RFD_API_CLIENT_SECRET} + # RFD_API_GITHUB_CALLBACK_URL: http://localhost:3000/auth/github/callback + # AUTH_PROVIDERS: github + # STORAGE_PROVIDER: s3 + # S3_BUCKET: ${S3_BUCKET} + # AWS_REGION: ${AWS_REGION} + # AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + # AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + # AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN} + # TELEMETRY_DISABLE: true + # GITHUB_HOST: ${GITHUB_HOST} + # GITHUB_REPO_CLIENT_ID: ${GITHUB_REPO_CLIENT_ID} + # GITHUB_REPO_CLIENT_SECRET: ${GITHUB_REPO_CLIENT_SECRET} + + rfd-migration: + build: + dockerfile: Containerfile + image: localhost/rfd-api-test:latest + networks: + - rfd + depends_on: + postgres: + condition: service_healthy + entrypoint: [] + working_dir: /home/rfd/db + environment: + - DATABASE_URL=postgres://rfd:rfd@postgres/rfd + command: /usr/local/bin/rfd-installer && diesel migration run && echo "Migrations Complete!" + + rfd-api: + image: localhost/rfd-api-test:latest + # Uses default CMD: rfd-api + ports: + - "8080:8080" + environment: + # - RFD_API_CONFIG=/config/rfd-api.toml + - DATABASE_URL=postgres://rfd:rfd@postgres/rfd + command: + - /usr/local/bin/rfd-api + - /config/rfd-api.toml + volumes: + - ./config:/config:ro + networks: + - rfd + depends_on: + postgres: + condition: service_started + meilisearch: + condition: service_started + rfd-migration: + condition: service_completed_successfully + + rfd-processor: + image: localhost/rfd-api-test:latest + + # Override CMD to run rfd-processor instead + environment: + # - RFD_API_CONFIG=/config/rfd-api.toml + - DATABASE_URL=postgres://rfd:rfd@postgres/rfd + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + command: + - /usr/local/bin/rfd-processor + - /config/rfd-processor.toml + volumes: + - ./config:/config:ro + networks: + - rfd + depends_on: + postgres: + condition: service_started + meilisearch: + condition: service_started + rfd-migration: + condition: service_completed_successfully + + postgres: + image: docker.io/postgres:14-trixie + environment: + POSTGRES_USER: rfd + POSTGRES_PASSWORD: rfd + POSTGRES_DB: rfd + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - "5432:5432" + networks: + - rfd + healthcheck: + test: ["CMD-SHELL", "pg_isready -U rfd"] + start_period: 5s + start_interval: 1s + retries: 10 + interval: 5s + + meilisearch: + image: docker.io/getmeili/meilisearch:v1.11 + environment: + MEILI_ENV: development + MEILI_MASTER_KEY: development-master-key + volumes: + - meilisearch_data:/meili_data + ports: + - "7700:7700" + networks: + - rfd + +networks: + rfd: + driver: bridge + +volumes: + postgres_data: + meilisearch_data: diff --git a/rfd-api/src/config.rs b/rfd-api/src/config.rs index 064ec64f..1179f41e 100644 --- a/rfd-api/src/config.rs +++ b/rfd-api/src/config.rs @@ -15,6 +15,7 @@ use crate::server::SpecConfig; #[derive(Debug, Deserialize)] pub struct AppConfig { pub log_format: ServerLogFormat, + pub log_filter: Option, pub log_directory: Option, pub initial_mappers: Option, pub public_url: String, diff --git a/rfd-api/src/main.rs b/rfd-api/src/main.rs index da4fd5f8..731934d4 100644 --- a/rfd-api/src/main.rs +++ b/rfd-api/src/main.rs @@ -55,10 +55,15 @@ async fn main() -> anyhow::Result<()> { NonBlocking::new(std::io::stdout()) }; + let env_filter = match config.log_filter { + Some(ref filter) => EnvFilter::new(filter), + None => EnvFilter::from_default_env(), + }; + let subscriber = tracing_subscriber::fmt() .with_file(false) .with_line_number(false) - .with_env_filter(EnvFilter::from_default_env()) + .with_env_filter(env_filter) .with_writer(writer); match config.log_format { diff --git a/rfd-api/src/server.rs b/rfd-api/src/server.rs index 70f19e04..61ae1f17 100644 --- a/rfd-api/src/server.rs +++ b/rfd-api/src/server.rs @@ -56,7 +56,7 @@ pub fn server( // Construct a shim to pipe dropshot logs into the global tracing logger let dropshot_logger = { - let level_drain = slog::LevelFilter(TracingSlogDrain, slog::Level::Debug).fuse(); + let level_drain = slog::LevelFilter(TracingSlogDrain, slog::Level::Trace).fuse(); let async_drain = slog_async::Async::new(level_drain).build().fuse(); slog::Logger::root(async_drain, slog::o!()) }; diff --git a/rfd-installer/src/lib.rs b/rfd-installer/src/lib.rs index e968fa6e..18c2a899 100644 --- a/rfd-installer/src/lib.rs +++ b/rfd-installer/src/lib.rs @@ -11,6 +11,7 @@ use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; const MIGRATIONS: EmbeddedMigrations = embed_migrations!("../rfd-model/migrations"); pub fn run_migrations(url: &str, v_only: bool) { + // These are safe to run multiple times. v_api_installer::run_migrations(url); if !v_only { diff --git a/rfd-model/diesel.toml b/rfd-model/diesel.toml index 2f2493e1..863549e3 100644 --- a/rfd-model/diesel.toml +++ b/rfd-model/diesel.toml @@ -1,9 +1,9 @@ # For documentation on how to configure this file, # see https://diesel.rs/guides/configuring-diesel-cli -[print_schema] -file = "src/schema.rs" -patch_file = "diesel-schema.patch" +# [print_schema] +# file = "src/schema.rs" +# patch_file = "diesel-schema.patch" [print_schema.filter] except_tables = [ diff --git a/rfd-processor/Cargo.toml b/rfd-processor/Cargo.toml index c853f700..178b9fb0 100644 --- a/rfd-processor/Cargo.toml +++ b/rfd-processor/Cargo.toml @@ -5,8 +5,13 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[dev-dependencies] +mockall = { workspace = true } + [dependencies] async-trait = { workspace = true } +aws-config = { workspace = true } +aws-sdk-s3 = { workspace = true } base64 = { workspace = true } chrono = { workspace = true } config = { workspace = true } diff --git a/rfd-processor/config.example.toml b/rfd-processor/config.example.toml index f33fbfed..385c7f38 100644 --- a/rfd-processor/config.example.toml +++ b/rfd-processor/config.example.toml @@ -69,10 +69,24 @@ path = "" # Branch to use as the default branch of the repository default_branch = "" -# Bucket to push static assets pulled from RFDs to (currently only GCP Storage buckets are supported) -[[static_storage]] -# Name of the bucket +# Static asset storage for images extracted from RFDs. Both GCS and S3 are supported. +# Multiple storage backends can be configured, and assets will be pushed to all of them. + +# Google Cloud Storage (GCS) +# Requires GCP Application Default Credentials to be configured +[[gcs_storage]] +# Name of the GCS bucket +bucket = "" + +# Amazon S3 (or S3-compatible storage) +# Uses AWS SDK default credential chain +[[s3_storage]] +# Name of the S3 bucket bucket = "" +# AWS region (e.g., "us-west-2") +region = "" +# Optional: Custom endpoint URL for S3-compatible services (e.g., MinIO, Backblaze B2) +# endpoint = "https://s3.example.com" # Location to store generated PDFs (currently on Google Drive Shared Drives are supported) [pdf_storage] diff --git a/rfd-processor/src/context.rs b/rfd-processor/src/context.rs index 7c5324b1..fbb6ce36 100644 --- a/rfd-processor/src/context.rs +++ b/rfd-processor/src/context.rs @@ -34,9 +34,22 @@ use crate::{ search::{RfdSearchIndex, SearchError}, updater::{BoxedAction, RfdUpdateMode, RfdUpdaterError}, util::{gdrive_client, GDriveError}, - AppConfig, GitHubAuthConfig, PdfStorageConfig, SearchConfig, StaticStorageConfig, + AppConfig, GcsStorageConfig, GitHubAuthConfig, PdfStorageConfig, S3StorageConfig, SearchConfig, }; +pub type StaticStorageError = Box; + +#[async_trait] +pub trait StaticStorage: Send + Sync { + async fn put( + &self, + key: &str, + data: Vec, + content_type: &str, + ) -> Result<(), StaticStorageError>; + fn name(&self) -> &str; +} + pub struct Database { pub storage: PostgresStore, } @@ -80,8 +93,8 @@ pub struct Context { pub db: Database, pub github: GitHubCtx, pub actions: Vec, - pub assets: StaticAssetStorageCtx, - pub pdf: PdfStorageCtx, + pub static_storage: Vec>, + pub pdf: Option, pub search: SearchCtx, } @@ -161,7 +174,7 @@ impl Context { .iter() .map(|action| action.as_str().try_into()) .collect::, RfdUpdaterError>>()?, - assets: StaticAssetStorageCtx::new(&config.static_storage).await?, + static_storage: build_static_storage(&config.gcs_storage, &config.s3_storage).await?, pdf: PdfStorageCtx::new(&config.pdf_storage).await?, search: SearchCtx::new(&config.search_storage)?, }) @@ -186,69 +199,150 @@ pub struct GitHubCtx { pub repository: GitHubRfdRepo, } -pub struct StaticAssetStorageCtx { - pub client: Storage>, - pub locations: Vec, +async fn build_static_storage( + gcs_entries: &[GcsStorageConfig], + s3_entries: &[S3StorageConfig], +) -> Result>, ContextError> { + let mut storage: Vec> = Vec::new(); + + // Build GCS storage instances + for entry in gcs_entries { + let client = build_gcs_client().await?; + storage.push(Box::new(GcsStorage { + client, + bucket: entry.bucket.clone(), + })); + } + + // Build S3 storage instances + for entry in s3_entries { + let mut config_loader = aws_config::defaults(aws_config::BehaviorVersion::latest()) + .region(aws_config::Region::new(entry.region.clone())); + + if let Some(endpoint) = &entry.endpoint { + config_loader = config_loader.endpoint_url(endpoint); + } + + let sdk_config = config_loader.load().await; + let client = aws_sdk_s3::Client::new(&sdk_config); + + storage.push(Box::new(S3Storage { + client, + bucket: entry.bucket.clone(), + })); + } + + Ok(storage) } -impl StaticAssetStorageCtx { - pub async fn new(entries: &[StaticStorageConfig]) -> Result { - let opts = yup_oauth2::ApplicationDefaultCredentialsFlowOpts::default(); - let gcp_auth = match yup_oauth2::ApplicationDefaultCredentialsAuthenticator::builder(opts) - .await - { - yup_oauth2::authenticator::ApplicationDefaultCredentialsTypes::ServiceAccount(auth) => { - tracing::debug!("Service account based credentials"); - - auth.build().await.map_err(|err| { - tracing::error!( - ?err, - "Failed to construct Cloud Storage credentials from service account" - ); - ContextError::FailedToFindGcpCredentials(err) - })? - } - yup_oauth2::authenticator::ApplicationDefaultCredentialsTypes::InstanceMetadata( - auth, - ) => { - tracing::debug!("Create instance based credentials"); - - auth.build().await.map_err(|err| { - tracing::error!( - ?err, - "Failed to construct Cloud Storage credentials from instance metadata" - ); - ContextError::FailedToFindGcpCredentials(err) - })? - } - }; +async fn build_gcs_client() -> Result>, ContextError> { + let opts = yup_oauth2::ApplicationDefaultCredentialsFlowOpts::default(); + let gcp_auth = match yup_oauth2::ApplicationDefaultCredentialsAuthenticator::builder(opts).await + { + yup_oauth2::authenticator::ApplicationDefaultCredentialsTypes::ServiceAccount(auth) => { + tracing::debug!("Service account based credentials"); + + auth.build().await.map_err(|err| { + tracing::error!( + ?err, + "Failed to construct Cloud Storage credentials from service account" + ); + ContextError::FailedToFindGcpCredentials(err) + })? + } + yup_oauth2::authenticator::ApplicationDefaultCredentialsTypes::InstanceMetadata(auth) => { + tracing::debug!("Create instance based credentials"); + + auth.build().await.map_err(|err| { + tracing::error!( + ?err, + "Failed to construct Cloud Storage credentials from instance metadata" + ); + ContextError::FailedToFindGcpCredentials(err) + })? + } + }; + + Ok(Storage::new( + Client::builder(TokioExecutor::new()).build( + HttpsConnectorBuilder::new() + .with_native_roots() + .unwrap() + .https_only() + .enable_http2() + .build(), + ), + gcp_auth, + )) +} - let storage = Storage::new( - Client::builder(TokioExecutor::new()).build( - HttpsConnectorBuilder::new() - .with_native_roots() - .unwrap() - .https_only() - .enable_http2() - .build(), - ), - gcp_auth, - ); +pub struct GcsStorage { + client: Storage>, + bucket: String, +} - Ok(Self { - client: storage, - locations: entries - .iter() - .map(|e| StaticAssetLocation { - bucket: e.bucket.to_string(), - }) - .collect(), - }) +#[async_trait] +impl StaticStorage for GcsStorage { + async fn put( + &self, + key: &str, + data: Vec, + content_type: &str, + ) -> Result<(), StaticStorageError> { + use google_storage1::api::Object; + + let mime_type: mime_guess::Mime = content_type + .parse() + .unwrap_or(mime_guess::mime::APPLICATION_OCTET_STREAM); + let cursor = std::io::Cursor::new(data); + + self.client + .objects() + .insert(Object::default(), &self.bucket) + .name(key) + .upload(cursor, mime_type) + .await?; + + Ok(()) + } + + fn name(&self) -> &str { + &self.bucket } } -pub struct StaticAssetLocation { - pub bucket: String, +pub struct S3Storage { + client: aws_sdk_s3::Client, + bucket: String, +} + +#[async_trait] +impl StaticStorage for S3Storage { + async fn put( + &self, + key: &str, + data: Vec, + content_type: &str, + ) -> Result<(), StaticStorageError> { + use aws_sdk_s3::primitives::ByteStream; + + let body = ByteStream::from(data); + + self.client + .put_object() + .bucket(&self.bucket) + .key(key) + .content_type(content_type) + .body(body) + .send() + .await?; + + Ok(()) + } + + fn name(&self) -> &str { + &self.bucket + } } pub type GDriveClient = DriveHub>; @@ -259,19 +353,16 @@ pub struct PdfStorageCtx { } impl PdfStorageCtx { - pub async fn new(config: &Option) -> Result { - Ok(Self { - // A client is only needed if files are going to be written - client: gdrive_client().await?, - locations: config - .as_ref() - .map(|config| { - vec![PdfStorageLocation { - folder_id: config.folder.clone(), - }] - }) - .unwrap_or_default(), - }) + pub async fn new(config: &Option) -> Result, GDriveError> { + match config { + Some(config) => Ok(Some(Self { + client: gdrive_client().await?, + locations: vec![PdfStorageLocation { + folder_id: config.folder.clone(), + }], + })), + None => Ok(None), + } } } @@ -324,7 +415,7 @@ impl PdfStorage for PdfStorageCtx { } } .tap_ok(|_| { - tracing::info!("Sucessfully uploaded PDF"); + tracing::info!("Successfully uploaded PDF"); }) .tap_err(|err| { tracing::error!(?err, "Failed to upload PDF"); @@ -364,3 +455,144 @@ impl SearchCtx { }) } } + +#[cfg(test)] +pub mod test { + use super::*; + use mockall::mock; + use std::collections::HashMap; + use std::sync::{Arc, Mutex}; + + mock! { + pub StaticStorage {} + + #[async_trait] + impl StaticStorage for StaticStorage { + async fn put( + &self, + key: &str, + data: Vec, + content_type: &str, + ) -> Result<(), StaticStorageError>; + fn name(&self) -> &str; + } + } + + /// In-memory storage implementation for testing + pub struct InMemoryStorage { + name: String, + objects: Arc>>, + } + + #[derive(Clone)] + pub struct StoredObject { + pub data: Vec, + pub content_type: String, + } + + impl InMemoryStorage { + pub fn new(name: &str) -> Self { + Self { + name: name.to_string(), + objects: Arc::new(Mutex::new(HashMap::new())), + } + } + + pub fn get(&self, key: &str) -> Option { + self.objects.lock().unwrap().get(key).cloned() + } + + pub fn len(&self) -> usize { + self.objects.lock().unwrap().len() + } + } + + #[async_trait] + impl StaticStorage for InMemoryStorage { + async fn put( + &self, + key: &str, + data: Vec, + content_type: &str, + ) -> Result<(), StaticStorageError> { + self.objects.lock().unwrap().insert( + key.to_string(), + StoredObject { + data, + content_type: content_type.to_string(), + }, + ); + Ok(()) + } + + fn name(&self) -> &str { + &self.name + } + } + + #[tokio::test] + async fn in_memory_storage_stores_and_retrieves_objects() { + let storage = InMemoryStorage::new("test-bucket"); + + let data = b"hello world".to_vec(); + storage + .put("test/key.txt", data.clone(), "text/plain") + .await + .unwrap(); + + let obj = storage.get("test/key.txt").unwrap(); + assert_eq!(obj.data, data); + assert_eq!(obj.content_type, "text/plain"); + } + + #[tokio::test] + async fn in_memory_storage_returns_correct_name() { + let storage = InMemoryStorage::new("my-bucket"); + assert_eq!(storage.name(), "my-bucket"); + } + + #[tokio::test] + async fn in_memory_storage_overwrites_existing_objects() { + let storage = InMemoryStorage::new("test-bucket"); + + storage + .put("key", b"first".to_vec(), "text/plain") + .await + .unwrap(); + storage + .put("key", b"second".to_vec(), "text/plain") + .await + .unwrap(); + + let obj = storage.get("key").unwrap(); + assert_eq!(obj.data, b"second".to_vec()); + assert_eq!(storage.len(), 1); + } + + #[tokio::test] + async fn mock_storage_can_simulate_failure() { + let mut mock = MockStaticStorage::new(); + mock.expect_put() + .returning(|_, _, _| Err("simulated failure".into())); + mock.expect_name().return_const("mock-bucket".to_string()); + + let result = mock.put("key", vec![], "text/plain").await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn mock_storage_can_verify_calls() { + let mut mock = MockStaticStorage::new(); + mock.expect_put() + .withf(|key, data, content_type| { + key == "expected/key.png" && data == b"image data" && content_type == "image/png" + }) + .times(1) + .returning(|_, _, _| Ok(())); + mock.expect_name().return_const("mock-bucket".to_string()); + + mock.put("expected/key.png", b"image data".to_vec(), "image/png") + .await + .unwrap(); + } +} diff --git a/rfd-processor/src/main.rs b/rfd-processor/src/main.rs index 0641d2b5..895ff0ab 100644 --- a/rfd-processor/src/main.rs +++ b/rfd-processor/src/main.rs @@ -31,6 +31,7 @@ mod util; #[derive(Debug, Deserialize)] pub struct AppConfig { pub log_directory: Option, + pub log_filter: Option, #[serde(default)] pub log_format: LogFormat, pub processor_enabled: bool, @@ -45,7 +46,9 @@ pub struct AppConfig { pub auth: AuthConfig, pub source: GitHubSourceRepo, #[serde(default)] - pub static_storage: Vec, + pub gcs_storage: Vec, + #[serde(default)] + pub s3_storage: Vec, #[serde(default)] pub pdf_storage: Option, #[serde(default)] @@ -96,10 +99,17 @@ pub struct GitHubSourceRepo { } #[derive(Debug, Deserialize, Serialize)] -pub struct StaticStorageConfig { +pub struct GcsStorageConfig { pub bucket: String, } +#[derive(Debug, Deserialize, Serialize)] +pub struct S3StorageConfig { + pub bucket: String, + pub region: String, + pub endpoint: Option, +} + #[derive(Debug, Deserialize, Serialize)] pub struct PdfStorageConfig { pub drive: Option, @@ -145,10 +155,15 @@ async fn main() -> Result<(), Box> { NonBlocking::new(std::io::stdout()) }; + let env_filter = match config.log_filter { + Some(ref filter) => EnvFilter::new(filter), + None => EnvFilter::from_default_env(), + }; + let subscriber = tracing_subscriber::fmt() .with_file(false) .with_line_number(false) - .with_env_filter(EnvFilter::from_default_env()) + .with_env_filter(env_filter) .with_writer(writer); match config.log_format { diff --git a/rfd-processor/src/updater/copy_images_to_storage.rs b/rfd-processor/src/updater/copy_images_to_storage.rs index ee72a669..73a7768f 100644 --- a/rfd-processor/src/updater/copy_images_to_storage.rs +++ b/rfd-processor/src/updater/copy_images_to_storage.rs @@ -3,7 +3,6 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use async_trait::async_trait; -use google_storage1::api::Object; use tracing::instrument; use crate::{rfd::PersistedRfd, util::decode_base64}; @@ -49,24 +48,25 @@ impl RfdUpdateAction for CopyImagesToStorage { "Writing file to storage buckets" ); - let cursor = std::io::Cursor::new(data); - - for location in &ctx.assets.locations { - tracing::info!(bucket = ?location.bucket, ?object_name, "Writing to location"); + for storage in &ctx.static_storage { + tracing::info!(name = storage.name(), ?object_name, "Writing to storage"); if mode == RfdUpdateMode::Write { - // TODO: Move implementation to a trait and abstract over different storage systems - if let Err(err) = ctx - .assets - .client - .objects() - .insert(Object::default(), &location.bucket) - .name(&object_name) - .upload(cursor.clone(), mime_type.clone()) + if let Err(err) = storage + .put(&object_name, data.clone(), mime_type.as_ref()) .await { - tracing::error!(?err, "Failed to upload static file to GCP"); + tracing::error!( + name = storage.name(), + ?err, + "Failed to upload static file" + ); } + } else { + tracing::warn!( + "CopyImagesToStorage is enabled however RfdUpdateMode is not write: {:?}", + mode + ); } } } diff --git a/rfd-processor/src/updater/update_pdfs.rs b/rfd-processor/src/updater/update_pdfs.rs index 365dcac0..1c20d992 100644 --- a/rfd-processor/src/updater/update_pdfs.rs +++ b/rfd-processor/src/updater/update_pdfs.rs @@ -68,15 +68,21 @@ impl UpdatePdfs { let store_results = match mode { RfdUpdateMode::Read => Vec::new(), - RfdUpdateMode::Write => { - ctx.pdf - .store_rfd_pdf( - new.pdf_external_id.as_deref(), - &new.get_pdf_filename(), - &pdf, - ) - .await - } + RfdUpdateMode::Write => match &ctx.pdf { + Some(pdf_storage) => { + pdf_storage + .store_rfd_pdf( + new.pdf_external_id.as_deref(), + &new.get_pdf_filename(), + &pdf, + ) + .await + } + None => { + tracing::debug!("PDF storage is disabled, skipping upload"); + Vec::new() + } + }, }; Ok(store_results From b4c8828ac951f1bfc6e5bbcfbafbdbb72cd0eefa Mon Sep 17 00:00:00 2001 From: Nick Downs Date: Fri, 6 Feb 2026 11:06:30 -0800 Subject: [PATCH 2/2] Add node and disable meilisearch analytics --- Containerfile | 16 ++++++++++++---- compose.yml | 2 ++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Containerfile b/Containerfile index 3c0dcfaf..d31ed052 100644 --- a/Containerfile +++ b/Containerfile @@ -2,7 +2,8 @@ # Build stage FROM docker.io/rust:1-trixie AS builder -ARG DIESEL_VER=v2.3.5 +ARG DIESEL_VERSION=v2.3.5 +ARG NODE_VERSION=v24.13.0 WORKDIR /app @@ -11,7 +12,6 @@ RUN apt-get update && apt-get install -y \ libpq-dev \ pkg-config - # Copy workspace files COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ @@ -39,10 +39,15 @@ RUN cargo build --release \ # Download diesel tool for migrations WORKDIR /tmp -RUN curl -L --output-dir /tmp -O "https://github.com/diesel-rs/diesel/releases/download/${DIESEL_VER}/diesel_cli-x86_64-unknown-linux-gnu.tar.xz" \ - && curl -L --output-dir /tmp -O "https://github.com/diesel-rs/diesel/releases/download/${DIESEL_VER}/diesel_cli-x86_64-unknown-linux-gnu.tar.xz.sha256" \ +RUN curl -L --output-dir /tmp -O "https://github.com/diesel-rs/diesel/releases/download/${DIESEL_VERSION}/diesel_cli-x86_64-unknown-linux-gnu.tar.xz" \ + && curl -L --output-dir /tmp -O "https://github.com/diesel-rs/diesel/releases/download/${DIESEL_VERSION}/diesel_cli-x86_64-unknown-linux-gnu.tar.xz.sha256" \ && sha256sum diesel_cli-x86_64-unknown-linux-gnu.tar.xz.sha256 && tar --strip-components=1 -xJvf diesel_cli-x86_64-unknown-linux-gnu.tar.xz +# Node for search indec +RUN curl -L --output-dir /tmp -O "https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-x64.tar.xz" \ + && curl -L --output-dir /tmp -o node.SHASUMS256.txt "https://nodejs.org/dist/${NODE_VERSION}/SHASUMS256.txt" \ + && sha256sum node.SHASUMS256.txt && tar --strip-components=1 -xJvf "node-${NODE_VERSION}-linux-x64.tar.xz" + # Runtime stage FROM docker.io/debian:trixie-slim @@ -67,6 +72,9 @@ COPY --from=builder /tmp/diesel /usr/local/bin/ # COPY --from=builder /app/rfd-model/diesel.toml /home/rfd/db/ COPY --from=builder /app/rfd-model/migrations/ /home/rfd/db/migrations/ +# Node for search indexing +COPY --from=builder /tmp/bin/node /usr/local/bin/ + # Create non-root user USER rfd WORKDIR /home/rfd diff --git a/compose.yml b/compose.yml index 136ff31c..8378a8dc 100644 --- a/compose.yml +++ b/compose.yml @@ -115,6 +115,8 @@ services: environment: MEILI_ENV: development MEILI_MASTER_KEY: development-master-key + MEILI_NO_ANALYTICS: "true" + MEILI_DB_PATH: /meili_data/data.ms volumes: - meilisearch_data:/meili_data ports: