From cc166e877c56a85166cfa578e85671cb772f2a15 Mon Sep 17 00:00:00 2001 From: Chase Douglas Date: Tue, 5 May 2026 13:49:00 -0700 Subject: [PATCH] storage: add OpenDAL S3 URI options OpenDAL S3 storage accepts bucket and root path data today, but serverless deployments also need URI query parameters to describe provider behavior in one DATA_FOLDER value. Update OpenDAL to 0.56.0 and build S3 operators with S3Config::from_uri(). Keep Vaultwarden's AWS SDK credential chain by installing a reqsign provider when the URI does not explicitly request OpenDAL-native credential handling. Move path handling and operator construction into storage.rs so S3-specific parsing, credential setup, and URI path manipulation stay out of configuration handling. Local filesystem behavior is unchanged, and S3 child paths are derived before query strings. --- Cargo.lock | 517 ++++++++++++++++++++++++------------ Cargo.toml | 8 +- src/api/core/sends.rs | 2 +- src/auth.rs | 8 +- src/config.rs | 131 ++------- src/db/models/attachment.rs | 2 +- src/db/models/send.rs | 2 +- src/main.rs | 1 + src/storage.rs | 297 +++++++++++++++++++++ 9 files changed, 678 insertions(+), 290 deletions(-) create mode 100644 src/storage.rs diff --git a/Cargo.lock b/Cargo.lock index ac84b501..959deb46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,17 +8,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures 0.2.17", -] - [[package]] name = "ahash" version = "0.8.12" @@ -391,6 +380,28 @@ dependencies = [ "zeroize", ] +[[package]] +name = "aws-lc-rs" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "aws-runtime" version = "1.7.3" @@ -670,17 +681,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "backon" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" -dependencies = [ - "fastrand", - "gloo-timers", - "tokio", -] - [[package]] name = "base16ct" version = "0.2.0" @@ -778,15 +778,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "block-padding" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" -dependencies = [ - "generic-array", -] - [[package]] name = "blocking" version = "1.6.2" @@ -892,15 +883,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ade8366b8bd5ba243f0a58f036cc0ca8a2f069cff1a2351ef1cac6b083e16fc0" -[[package]] -name = "cbc" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" -dependencies = [ - "cipher", -] - [[package]] name = "cc" version = "1.2.61" @@ -913,6 +895,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.4" @@ -961,13 +949,12 @@ dependencies = [ ] [[package]] -name = "cipher" -version = "0.4.4" +name = "cmake" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" dependencies = [ - "crypto-common 0.1.6", - "inout", + "cc", ] [[package]] @@ -1668,6 +1655,12 @@ dependencies = [ "syn", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -1916,6 +1909,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.32" @@ -2275,7 +2274,7 @@ dependencies = [ "hickory-proto", "idna", "ipnet", - "jni", + "jni 0.22.4", "rand 0.10.1", "thiserror 2.0.18", "tinyvec", @@ -2293,7 +2292,7 @@ dependencies = [ "data-encoding", "idna", "ipnet", - "jni", + "jni 0.22.4", "once_cell", "prefix-trie", "rand 0.10.1", @@ -2316,7 +2315,7 @@ dependencies = [ "hickory-proto", "ipconfig", "ipnet", - "jni", + "jni 0.22.4", "moka", "ndk-context", "once_cell", @@ -2357,15 +2356,6 @@ dependencies = [ "digest 0.11.2", ] -[[package]] -name = "home" -version = "0.5.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" -dependencies = [ - "windows-sys 0.61.2", -] - [[package]] name = "hostname" version = "0.4.2" @@ -2716,16 +2706,6 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" -[[package]] -name = "inout" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" -dependencies = [ - "block-padding", - "generic-array", -] - [[package]] name = "ipconfig" version = "0.3.4" @@ -2798,10 +2778,12 @@ checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" dependencies = [ "jiff-static", "jiff-tzdb-platform", + "js-sys", "log", "portable-atomic", "portable-atomic-util", "serde_core", + "wasm-bindgen", "windows-sys 0.61.2", ] @@ -2831,6 +2813,22 @@ dependencies = [ "jiff-tzdb", ] +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + [[package]] name = "jni" version = "0.22.4" @@ -2840,7 +2838,7 @@ dependencies = [ "cfg-if", "combine", "jni-macros", - "jni-sys", + "jni-sys 0.4.1", "log", "simd_cesu8", "thiserror 2.0.18", @@ -2861,6 +2859,15 @@ dependencies = [ "syn", ] +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + [[package]] name = "jni-sys" version = "0.4.1" @@ -2913,21 +2920,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "jsonwebtoken" -version = "9.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" -dependencies = [ - "base64 0.22.1", - "js-sys", - "pem", - "ring", - "serde", - "serde_json", - "simple_asn1", -] - [[package]] name = "jsonwebtoken" version = "10.3.0" @@ -3131,6 +3123,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "mea" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6747f54621d156e1b47eb6b25f39a941b9fc347f98f67d25d8881ff99e8ed832" +dependencies = [ + "slab", +] + [[package]] name = "memchr" version = "2.8.0" @@ -3408,7 +3409,7 @@ dependencies = [ "getrandom 0.2.17", "http 1.4.0", "rand 0.8.6", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "serde_path_to_error", @@ -3438,31 +3439,76 @@ dependencies = [ [[package]] name = "opendal" -version = "0.55.0" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d075ab8a203a6ab4bc1bce0a4b9fe486a72bf8b939037f4b78d95386384bc80a" +checksum = "97b31d3d8e99a85d83b73ec26647f5607b80578ed9375810b6e44ffa3590a236" +dependencies = [ + "opendal-core", + "opendal-service-fs", + "opendal-service-s3", +] + +[[package]] +name = "opendal-core" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1849dd2687e173e776d3af5fce1ba3ae47b9dd37a09d1c4deba850ef45fe00ca" dependencies = [ "anyhow", - "backon", "base64 0.22.1", "bytes", - "crc32c", "futures", - "getrandom 0.2.17", "http 1.4.0", "http-body 1.0.1", "jiff", "log", "md-5", + "mea", "percent-encoding", "quick-xml 0.38.4", - "reqsign", - "reqwest", + "reqsign-core", + "reqwest 0.13.3", "serde", "serde_json", "tokio", "url", "uuid", + "web-time", +] + +[[package]] +name = "opendal-service-fs" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf0be0417abeeb0053376d816b90fceb9ca98f20dfb54ebf1f2a282729f83663" +dependencies = [ + "bytes", + "log", + "opendal-core", + "serde", + "tokio", + "xattr", +] + +[[package]] +name = "opendal-service-s3" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dadddeb9bb50b0d30927dd914c298c4ddca47e4c1cfa7674d311f0cf9b051c8" +dependencies = [ + "base64 0.22.1", + "bytes", + "crc32c", + "http 1.4.0", + "log", + "md-5", + "opendal-core", + "quick-xml 0.38.4", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "url", ] [[package]] @@ -3651,16 +3697,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5a797f0e07bdf071d15742978fc3128ec6c22891c31a3a931513263904c982a" -[[package]] -name = "pbkdf2" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" -dependencies = [ - "digest 0.10.7", - "hmac 0.12.1", -] - [[package]] name = "pear" version = "0.2.9" @@ -3852,21 +3888,6 @@ dependencies = [ "spki", ] -[[package]] -name = "pkcs5" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" -dependencies = [ - "aes", - "cbc", - "der", - "pbkdf2", - "scrypt", - "sha2 0.10.9", - "spki", -] - [[package]] name = "pkcs8" version = "0.10.2" @@ -3874,8 +3895,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ "der", - "pkcs5", - "rand_core 0.6.4", "spki", ] @@ -4040,9 +4059,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quick-xml" -version = "0.37.5" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" dependencies = [ "memchr", "serde", @@ -4050,9 +4069,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.38.4" +version = "0.39.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +checksum = "721da970c312655cde9b4ffe0547f20a8494866a4af5ff51f18b7c633d0c870b" dependencies = [ "memchr", "serde", @@ -4084,6 +4103,7 @@ version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", @@ -4312,36 +4332,57 @@ dependencies = [ ] [[package]] -name = "reqsign" -version = "0.16.5" +name = "reqsign-aws-v4" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43451dbf3590a7590684c25fb8d12ecdcc90ed3ac123433e500447c7d77ed701" +checksum = "44eaca382e94505a49f1a4849658d153aebf79d9c1a58e5dd3b10361511e9f43" dependencies = [ "anyhow", - "async-trait", - "base64 0.22.1", - "chrono", + "bytes", "form_urlencoded", - "getrandom 0.2.17", - "hex", - "hmac 0.12.1", - "home", "http 1.4.0", - "jsonwebtoken 9.3.1", "log", - "once_cell", "percent-encoding", - "quick-xml 0.37.5", - "rand 0.8.6", - "reqwest", - "rsa", + "quick-xml 0.39.3", + "reqsign-core", "rust-ini", "serde", "serde_json", + "serde_urlencoded", + "sha1", +] + +[[package]] +name = "reqsign-core" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b10302cf0a7d7e7352ba211fc92c3c5bebf1286153e49cc5aa87348078a8e102" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bytes", + "form_urlencoded", + "futures", + "hex", + "hmac 0.12.1", + "http 1.4.0", + "jiff", + "log", + "percent-encoding", "sha1", "sha2 0.10.9", + "windows-sys 0.61.2", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d89295b3d17abea31851cc8de55d843d89c52132c864963c38d41920613dc5" +dependencies = [ + "anyhow", + "reqsign-core", "tokio", - "toml 0.8.23", ] [[package]] @@ -4387,11 +4428,49 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.4.2", "web-sys", "webpki-roots", ] +[[package]] +name = "reqwest" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.9.0", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls 0.23.40", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls 0.26.4", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", + "web-sys", +] + [[package]] name = "resolv-conf" version = "0.7.6" @@ -4560,7 +4639,6 @@ dependencies = [ "pkcs1", "pkcs8", "rand_core 0.6.4", - "sha2 0.10.9", "signature", "spki", "subtle", @@ -4652,6 +4730,7 @@ version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -4692,6 +4771,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni 0.21.1", + "log", + "once_cell", + "rustls 0.23.40", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki 0.103.13", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -4708,6 +4814,7 @@ version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -4725,15 +4832,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "salsa20" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" -dependencies = [ - "cipher", -] - [[package]] name = "same-file" version = "1.0.6" @@ -4797,17 +4895,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "scrypt" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" -dependencies = [ - "pbkdf2", - "salsa20", - "sha2 0.10.9", -] - [[package]] name = "sct" version = "0.7.1" @@ -5869,7 +5956,6 @@ checksum = "7ba6f5989077681266825251a52748b8c1d8a4ad098cc37e440103d0ea717fc0" name = "vaultwarden" version = "1.0.0" dependencies = [ - "anyhow", "argon2", "aws-config", "aws-credential-types", @@ -5899,7 +5985,7 @@ dependencies = [ "html5gum", "http 1.4.0", "job_scheduler_ng", - "jsonwebtoken 10.3.0", + "jsonwebtoken", "lettre", "libsqlite3-sys", "log", @@ -5916,8 +6002,9 @@ dependencies = [ "pico-args", "rand 0.10.1", "regex", - "reqsign", - "reqwest", + "reqsign-aws-v4", + "reqsign-core", + "reqwest 0.12.28", "ring", "rmpv", "rocket", @@ -6094,6 +6181,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -6194,6 +6294,15 @@ dependencies = [ "url", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "1.0.7" @@ -6328,6 +6437,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -6364,6 +6482,21 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -6412,6 +6545,12 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -6430,6 +6569,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -6448,6 +6593,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -6478,6 +6629,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -6496,6 +6653,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -6514,6 +6677,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -6532,6 +6701,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -6691,6 +6866,16 @@ dependencies = [ "time", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "xml" version = "1.2.1" @@ -6746,7 +6931,7 @@ dependencies = [ "futures", "hmac 0.12.1", "rand 0.9.4", - "reqwest", + "reqwest 0.12.28", "sha1", "threadpool", ] diff --git a/Cargo.toml b/Cargo.toml index e7fd5ade..019eaf31 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,7 +39,7 @@ vendored_openssl = ["openssl/vendored"] # Enable MiMalloc memory allocator to replace the default malloc # This can improve performance for Alpine builds enable_mimalloc = ["dep:mimalloc"] -s3 = ["opendal/services-s3", "dep:aws-config", "dep:aws-credential-types", "dep:aws-smithy-runtime-api", "dep:anyhow", "dep:http", "dep:reqsign"] +s3 = ["opendal/services-s3", "opendal/reqwest-rustls-tls", "dep:aws-config", "dep:aws-credential-types", "dep:aws-smithy-runtime-api", "dep:http", "dep:reqsign-aws-v4", "dep:reqsign-core"] # OIDC specific features oidc-accept-rfc3339-timestamps = ["openidconnect/accept-rfc3339-timestamps"] @@ -196,15 +196,15 @@ rpassword = "7.5.1" grass_compiler = { version = "0.13.4", default-features = false } # File are accessed through Apache OpenDAL -opendal = { version = "0.55.0", features = ["services-fs"], default-features = false } +opendal = { version = "0.56.0", features = ["services-fs"], default-features = false } # For retrieving AWS credentials, including temporary SSO credentials -anyhow = { version = "1.0.102", optional = true } aws-config = { version = "1.8.16", features = ["behavior-version-latest", "rt-tokio", "credentials-process", "sso"], default-features = false, optional = true } aws-credential-types = { version = "1.2.14", optional = true } aws-smithy-runtime-api = { version = "1.12.0", optional = true } http = { version = "1.4.0", optional = true } -reqsign = { version = "0.16.5", optional = true } +reqsign-aws-v4 = { version = "3.0.0", optional = true } +reqsign-core = { version = "3.0.0", optional = true } # Strip debuginfo from the release builds # The debug symbols are to provide better panic traces diff --git a/src/api/core/sends.rs b/src/api/core/sends.rs index 22abb396..45ead810 100644 --- a/src/api/core/sends.rs +++ b/src/api/core/sends.rs @@ -568,7 +568,7 @@ async fn post_access_file( async fn download_url(host: &Host, send_id: &SendId, file_id: &SendFileId) -> Result { let operator = CONFIG.opendal_operator_for_path_type(&PathType::Sends)?; - if operator.info().scheme() == <&'static str>::from(opendal::Scheme::Fs) { + if crate::storage::is_fs_operator(&operator) { let token_claims = crate::auth::generate_send_claims(send_id, file_id); let token = crate::auth::encode_jwt(&token_claims); diff --git a/src/auth.rs b/src/auth.rs index 43184369..06bd9c22 100644 --- a/src/auth.rs +++ b/src/auth.rs @@ -54,12 +54,8 @@ static PUBLIC_RSA_KEY: OnceLock = OnceLock::new(); pub async fn initialize_keys() -> Result<(), Error> { use std::io::Error; - let rsa_key_filename = std::path::PathBuf::from(CONFIG.private_rsa_key()) - .file_name() - .ok_or_else(|| Error::other("Private RSA key path missing filename"))? - .to_str() - .ok_or_else(|| Error::other("Private RSA key path filename is not valid UTF-8"))? - .to_string(); + let rsa_key_filename = crate::storage::file_name(&CONFIG.private_rsa_key()) + .ok_or_else(|| Error::other("Private RSA key path missing filename"))?; let operator = CONFIG.opendal_operator_for_path_type(&PathType::RsaKey).map_err(Error::other)?; diff --git a/src/config.rs b/src/config.rs index ae995f69..b6d0ce8a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -14,6 +14,7 @@ use serde::de::{self, Deserialize, Deserializer, MapAccess, Visitor}; use crate::{ error::Error, + storage, util::{ get_active_web_release, get_env, get_env_bool, is_valid_email, parse_experimental_client_feature_flags, FeatureFlagFilter, @@ -22,18 +23,14 @@ use crate::{ static CONFIG_FILE: LazyLock = LazyLock::new(|| { let data_folder = get_env("DATA_FOLDER").unwrap_or_else(|| String::from("data")); - get_env("CONFIG_FILE").unwrap_or_else(|| format!("{data_folder}/config.json")) + get_env("CONFIG_FILE").unwrap_or_else(|| storage::join_path(&data_folder, "config.json")) }); -static CONFIG_FILE_PARENT_DIR: LazyLock = LazyLock::new(|| { - let path = std::path::PathBuf::from(&*CONFIG_FILE); - path.parent().unwrap_or(std::path::Path::new("data")).to_str().unwrap_or("data").to_string() -}); +static CONFIG_FILE_PARENT_DIR: LazyLock = + LazyLock::new(|| storage::parent(&CONFIG_FILE).unwrap_or_else(|| "data".to_string())); -static CONFIG_FILENAME: LazyLock = LazyLock::new(|| { - let path = std::path::PathBuf::from(&*CONFIG_FILE); - path.file_name().unwrap_or(std::ffi::OsStr::new("config.json")).to_str().unwrap_or("config.json").to_string() -}); +static CONFIG_FILENAME: LazyLock = + LazyLock::new(|| storage::file_name(&CONFIG_FILE).unwrap_or_else(|| "config.json".to_string())); pub static SKIP_CONFIG_VALIDATION: AtomicBool = AtomicBool::new(false); @@ -263,7 +260,7 @@ macro_rules! make_config { } async fn from_file() -> Result { - let operator = opendal_operator_for_path(&CONFIG_FILE_PARENT_DIR)?; + let operator = storage::operator_for_path(&CONFIG_FILE_PARENT_DIR)?; let config_bytes = operator.read(&CONFIG_FILENAME).await?; println!("[INFO] Using saved config from `{}` for configuration.\n", *CONFIG_FILE); serde_json::from_slice(&config_bytes.to_vec()).map_err(Into::into) @@ -507,19 +504,19 @@ make_config! { /// Data folder |> Main data folder data_folder: String, false, def, "data".to_string(); /// Database URL - database_url: String, false, auto, |c| format!("{}/db.sqlite3", c.data_folder); + database_url: String, false, auto, |c| storage::join_path(&c.data_folder, "db.sqlite3"); /// Icon cache folder - icon_cache_folder: String, false, auto, |c| format!("{}/icon_cache", c.data_folder); + icon_cache_folder: String, false, auto, |c| storage::join_path(&c.data_folder, "icon_cache"); /// Attachments folder - attachments_folder: String, false, auto, |c| format!("{}/attachments", c.data_folder); + attachments_folder: String, false, auto, |c| storage::join_path(&c.data_folder, "attachments"); /// Sends folder - sends_folder: String, false, auto, |c| format!("{}/sends", c.data_folder); + sends_folder: String, false, auto, |c| storage::join_path(&c.data_folder, "sends"); /// Temp folder |> Used for storing temporary file uploads - tmp_folder: String, false, auto, |c| format!("{}/tmp", c.data_folder); + tmp_folder: String, false, auto, |c| storage::join_path(&c.data_folder, "tmp"); /// Templates folder - templates_folder: String, false, auto, |c| format!("{}/templates", c.data_folder); + templates_folder: String, false, auto, |c| storage::join_path(&c.data_folder, "templates"); /// Session JWT key - rsa_key_filename: String, false, auto, |c| format!("{}/rsa_key", c.data_folder); + rsa_key_filename: String, false, auto, |c| storage::join_path(&c.data_folder, "rsa_key"); /// Web vault folder web_vault_folder: String, false, def, "web-vault/".to_string(); }, @@ -1366,90 +1363,6 @@ fn smtp_convert_deprecated_ssl_options(smtp_ssl: Option, smtp_explicit_tls "starttls".to_string() } -fn opendal_operator_for_path(path: &str) -> Result { - // Cache of previously built operators by path - static OPERATORS_BY_PATH: LazyLock> = - LazyLock::new(dashmap::DashMap::new); - - if let Some(operator) = OPERATORS_BY_PATH.get(path) { - return Ok(operator.clone()); - } - - let operator = if path.starts_with("s3://") { - #[cfg(not(s3))] - return Err(opendal::Error::new(opendal::ErrorKind::ConfigInvalid, "S3 support is not enabled").into()); - - #[cfg(s3)] - opendal_s3_operator_for_path(path)? - } else { - let builder = opendal::services::Fs::default().root(path); - opendal::Operator::new(builder)?.finish() - }; - - OPERATORS_BY_PATH.insert(path.to_string(), operator.clone()); - - Ok(operator) -} - -#[cfg(s3)] -fn opendal_s3_operator_for_path(path: &str) -> Result { - use crate::http_client::aws::AwsReqwestConnector; - use aws_config::{default_provider::credentials::DefaultCredentialsChain, provider_config::ProviderConfig}; - - // This is a custom AWS credential loader that uses the official AWS Rust - // SDK config crate to load credentials. This ensures maximum compatibility - // with AWS credential configurations. For example, OpenDAL doesn't support - // AWS SSO temporary credentials yet. - struct OpenDALS3CredentialLoader {} - - #[async_trait] - impl reqsign::AwsCredentialLoad for OpenDALS3CredentialLoader { - async fn load_credential(&self, _client: reqwest::Client) -> anyhow::Result> { - use aws_credential_types::provider::ProvideCredentials as _; - use tokio::sync::OnceCell; - - static DEFAULT_CREDENTIAL_CHAIN: OnceCell = OnceCell::const_new(); - - let chain = DEFAULT_CREDENTIAL_CHAIN - .get_or_init(|| { - let reqwest_client = reqwest::Client::builder().build().unwrap(); - let connector = AwsReqwestConnector { - client: reqwest_client, - }; - - let conf = ProviderConfig::default().with_http_client(connector); - - DefaultCredentialsChain::builder().configure(conf).build() - }) - .await; - - let creds = chain.provide_credentials().await?; - - Ok(Some(reqsign::AwsCredential { - access_key_id: creds.access_key_id().to_string(), - secret_access_key: creds.secret_access_key().to_string(), - session_token: creds.session_token().map(|s| s.to_string()), - expires_in: creds.expiry().map(|expiration| expiration.into()), - })) - } - } - - const OPEN_DAL_S3_CREDENTIAL_LOADER: OpenDALS3CredentialLoader = OpenDALS3CredentialLoader {}; - - let url = Url::parse(path).map_err(|e| format!("Invalid path S3 URL path {path:?}: {e}"))?; - - let bucket = url.host_str().ok_or_else(|| format!("Missing Bucket name in data folder S3 URL {path:?}"))?; - - let builder = opendal::services::S3::default() - .customized_credential_load(Box::new(OPEN_DAL_S3_CREDENTIAL_LOADER)) - .enable_virtual_host_style() - .bucket(bucket) - .root(url.path()) - .default_storage_class("INTELLIGENT_TIERING"); - - Ok(opendal::Operator::new(builder)?.finish()) -} - pub enum PathType { Data, IconCache, @@ -1547,7 +1460,7 @@ impl Config { } //Save to file - let operator = opendal_operator_for_path(&CONFIG_FILE_PARENT_DIR)?; + let operator = storage::operator_for_path(&CONFIG_FILE_PARENT_DIR)?; operator.write(&CONFIG_FILENAME, config_str).await?; Ok(()) @@ -1612,7 +1525,7 @@ impl Config { } pub async fn delete_user_config(&self) -> Result<(), Error> { - let operator = opendal_operator_for_path(&CONFIG_FILE_PARENT_DIR)?; + let operator = storage::operator_for_path(&CONFIG_FILE_PARENT_DIR)?; operator.delete(&CONFIG_FILENAME).await?; // Empty user config @@ -1636,7 +1549,7 @@ impl Config { } pub fn private_rsa_key(&self) -> String { - format!("{}.pem", self.rsa_key_filename()) + storage::with_extension(&self.rsa_key_filename(), "pem") } pub fn mail_enabled(&self) -> bool { let inner = &self.inner.read().unwrap().config; @@ -1677,15 +1590,11 @@ impl Config { PathType::IconCache => self.icon_cache_folder(), PathType::Attachments => self.attachments_folder(), PathType::Sends => self.sends_folder(), - PathType::RsaKey => std::path::Path::new(&self.rsa_key_filename()) - .parent() - .ok_or_else(|| std::io::Error::other("Failed to get directory of RSA key file"))? - .to_str() - .ok_or_else(|| std::io::Error::other("Failed to convert RSA key file directory to UTF-8 string"))? - .to_string(), + PathType::RsaKey => storage::parent(&self.private_rsa_key()) + .ok_or_else(|| std::io::Error::other("Failed to get directory of RSA key file"))?, }; - opendal_operator_for_path(&path) + storage::operator_for_path(&path) } pub fn render_template(&self, name: &str, data: &T) -> Result { diff --git a/src/db/models/attachment.rs b/src/db/models/attachment.rs index 7611b927..dad081bd 100644 --- a/src/db/models/attachment.rs +++ b/src/db/models/attachment.rs @@ -46,7 +46,7 @@ impl Attachment { pub async fn get_url(&self, host: &str) -> Result { let operator = CONFIG.opendal_operator_for_path_type(&PathType::Attachments)?; - if operator.info().scheme() == <&'static str>::from(opendal::Scheme::Fs) { + if crate::storage::is_fs_operator(&operator) { let token = encode_jwt(&generate_file_download_claims(self.cipher_uuid.clone(), self.id.clone())); Ok(format!("{host}/attachments/{}/{}?token={token}", self.cipher_uuid, self.id)) } else { diff --git a/src/db/models/send.rs b/src/db/models/send.rs index 84802c54..5b6611fa 100644 --- a/src/db/models/send.rs +++ b/src/db/models/send.rs @@ -237,7 +237,7 @@ impl Send { if self.atype == SendType::File as i32 { let operator = CONFIG.opendal_operator_for_path_type(&PathType::Sends)?; - operator.remove_all(&self.uuid).await.ok(); + operator.delete_with(&self.uuid).recursive(true).await.ok(); } db_run! { conn: { diff --git a/src/main.rs b/src/main.rs index 60c5a593..96aa55b5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -57,6 +57,7 @@ mod mail; mod ratelimit; mod sso; mod sso_client; +mod storage; mod util; use crate::api::core::two_factor::duo_oidc::purge_duo_contexts; diff --git a/src/storage.rs b/src/storage.rs new file mode 100644 index 00000000..ada2a951 --- /dev/null +++ b/src/storage.rs @@ -0,0 +1,297 @@ +use std::sync::LazyLock; + +pub(crate) fn join_path(base: &str, child: &str) -> String { + #[cfg(s3)] + if s3::is_uri(base) { + return s3::join_path(base, child); + } + + let base = base.trim_end_matches('/'); + let child = child.trim_start_matches('/'); + if base.is_empty() { + child.to_string() + } else if child.is_empty() { + base.to_string() + } else { + format!("{base}/{child}") + } +} + +pub(crate) fn with_extension(path: &str, extension: &str) -> String { + let extension = extension.trim_start_matches('.'); + + #[cfg(s3)] + if s3::is_uri(path) { + return s3::with_extension(path, extension); + } + + format!("{path}.{extension}") +} + +pub(crate) fn parent(path: &str) -> Option { + #[cfg(s3)] + if s3::is_uri(path) { + return s3::parent(path); + } + + std::path::Path::new(path).parent()?.to_str().map(ToString::to_string) +} + +pub(crate) fn file_name(path: &str) -> Option { + #[cfg(s3)] + if s3::is_uri(path) { + return s3::file_name(path); + } + + std::path::Path::new(path).file_name()?.to_str().map(ToString::to_string) +} + +pub(crate) fn is_fs_operator(operator: &opendal::Operator) -> bool { + operator.info().scheme() == opendal::services::FS_SCHEME +} + +pub(crate) fn operator_for_path(path: &str) -> Result { + // Cache of previously built operators by path + static OPERATORS_BY_PATH: LazyLock> = + LazyLock::new(dashmap::DashMap::new); + + if let Some(operator) = OPERATORS_BY_PATH.get(path) { + return Ok(operator.clone()); + } + + let operator = if path.starts_with("s3://") { + #[cfg(not(s3))] + return Err(opendal::Error::new(opendal::ErrorKind::ConfigInvalid, "S3 support is not enabled").into()); + + #[cfg(s3)] + s3::operator_for_path(path)? + } else { + let builder = opendal::services::Fs::default().root(path); + opendal::Operator::new(builder)?.finish() + }; + + OPERATORS_BY_PATH.insert(path.to_string(), operator.clone()); + + Ok(operator) +} + +#[cfg(s3)] +mod s3 { + use reqwest::Url; + + use crate::error::Error; + + pub(super) fn is_uri(path: &str) -> bool { + path.starts_with("s3://") + } + + pub(super) fn join_path(base: &str, child: &str) -> String { + if let Ok(mut url) = Url::parse(base) { + let mut segments = path_segments(&url); + segments.extend(child.split('/').filter(|segment| !segment.is_empty()).map(ToString::to_string)); + set_path_segments(&mut url, &segments); + return url.to_string(); + } + + let base = base.trim_end_matches('/'); + let child = child.trim_start_matches('/'); + if base.is_empty() { + child.to_string() + } else if child.is_empty() { + base.to_string() + } else { + format!("{base}/{child}") + } + } + + pub(super) fn with_extension(path: &str, extension: &str) -> String { + if let Ok(mut url) = Url::parse(path) { + let mut segments = path_segments(&url); + if let Some(file_name) = segments.last_mut() { + file_name.push('.'); + file_name.push_str(extension); + set_path_segments(&mut url, &segments); + return url.to_string(); + } + } + + format!("{path}.{extension}") + } + + pub(super) fn parent(path: &str) -> Option { + if let Ok(mut url) = Url::parse(path) { + let mut segments = path_segments(&url); + segments.pop()?; + set_path_segments(&mut url, &segments); + return Some(url.to_string()); + } + + std::path::Path::new(path).parent()?.to_str().map(ToString::to_string) + } + + pub(super) fn file_name(path: &str) -> Option { + if let Ok(url) = Url::parse(path) { + return path_segments(&url).pop(); + } + + std::path::Path::new(path).file_name()?.to_str().map(ToString::to_string) + } + + fn path_segments(url: &Url) -> Vec { + url.path_segments() + .map(|segments| segments.filter(|segment| !segment.is_empty()).map(ToString::to_string).collect()) + .unwrap_or_default() + } + + fn set_path_segments(url: &mut Url, segments: &[String]) { + if segments.is_empty() { + url.set_path(""); + } else { + url.set_path(&format!("/{}", segments.join("/"))); + } + } + + pub(super) fn operator_for_path(path: &str) -> Result { + use crate::http_client::aws::AwsReqwestConnector; + use aws_config::{default_provider::credentials::DefaultCredentialsChain, provider_config::ProviderConfig}; + use opendal::Configurator; + use reqsign_aws_v4::Credential; + use reqsign_core::{Context, ProvideCredential, ProvideCredentialChain}; + + // This is a custom AWS credential loader that uses the official AWS Rust + // SDK config crate to load credentials. This ensures maximum compatibility + // with AWS credential configurations. For example, OpenDAL doesn't support + // AWS SSO temporary credentials yet. + #[derive(Debug)] + struct OpenDALS3CredentialProvider; + + impl ProvideCredential for OpenDALS3CredentialProvider { + type Credential = Credential; + + async fn provide_credential(&self, _ctx: &Context) -> reqsign_core::Result> { + use aws_credential_types::provider::ProvideCredentials as _; + use reqsign_core::time::Timestamp; + use tokio::sync::OnceCell; + + static DEFAULT_CREDENTIAL_CHAIN: OnceCell = OnceCell::const_new(); + + let chain = DEFAULT_CREDENTIAL_CHAIN + .get_or_init(|| { + let reqwest_client = reqwest::Client::builder().build().unwrap(); + let connector = AwsReqwestConnector { + client: reqwest_client, + }; + + let conf = ProviderConfig::default().with_http_client(connector); + + DefaultCredentialsChain::builder().configure(conf).build() + }) + .await; + + let creds = chain.provide_credentials().await.map_err(|e| { + reqsign_core::Error::unexpected("failed to load AWS credentials via AWS SDK").with_source(e) + })?; + + let expires_in = if let Some(expiration) = creds.expiry() { + let duration = expiration.duration_since(std::time::UNIX_EPOCH).map_err(|e| { + reqsign_core::Error::unexpected("AWS credential expiration is before the Unix epoch") + .with_source(e) + })?; + let seconds = i64::try_from(duration.as_secs()).map_err(|e| { + reqsign_core::Error::unexpected("AWS credential expiration is too large").with_source(e) + })?; + Some(Timestamp::from_second(seconds)?) + } else { + None + }; + + Ok(Some(Credential { + access_key_id: creds.access_key_id().to_string(), + secret_access_key: creds.secret_access_key().to_string(), + session_token: creds.session_token().map(|s| s.to_string()), + expires_in, + })) + } + } + + let uri = opendal::OperatorUri::new(path, std::iter::empty::<(String, String)>())?; + let mut config = opendal::services::S3Config::from_uri(&uri)?; + + if !uri_has_option(&uri, &["default_storage_class"]) { + config.default_storage_class = Some("INTELLIGENT_TIERING".to_string()); + } + + if !uri_has_option( + &uri, + &["enable_virtual_host_style", "aws_virtual_hosted_style_request", "virtual_hosted_style_request"], + ) { + config.enable_virtual_host_style = true; + } + + let use_aws_sdk_credentials = !uri_has_credential_options(&uri, &config); + let mut builder = config.into_builder(); + + if use_aws_sdk_credentials { + builder = + builder.credential_provider_chain(ProvideCredentialChain::new().push(OpenDALS3CredentialProvider)); + } + + Ok(opendal::Operator::new(builder)?.finish()) + } + + fn uri_has_option(uri: &opendal::OperatorUri, names: &[&str]) -> bool { + names.iter().any(|name| uri.options().contains_key(*name)) + } + + fn uri_has_credential_options(uri: &opendal::OperatorUri, config: &opendal::services::S3Config) -> bool { + config.access_key_id.is_some() + || config.secret_access_key.is_some() + || config.session_token.is_some() + || config.role_arn.is_some() + || config.external_id.is_some() + || config.role_session_name.is_some() + || uri_has_option(uri, &["allow_anonymous", "disable_config_load", "disable_ec2_metadata"]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn handles_local_paths() { + assert_eq!(join_path("data", "attachments"), "data/attachments"); + assert_eq!(with_extension("data/rsa_key", "pem"), "data/rsa_key.pem"); + assert_eq!(parent("data/rsa_key.pem").as_deref(), Some("data")); + assert_eq!(file_name("data/rsa_key.pem").as_deref(), Some("rsa_key.pem")); + } +} + +#[cfg(all(test, s3))] +mod s3_tests { + use super::*; + + #[test] + fn joins_s3_path_before_query_string() { + assert_eq!( + join_path("s3://bucket/base?region=us-west-2", "attachments"), + "s3://bucket/base/attachments?region=us-west-2" + ); + } + + #[test] + fn appends_extension_before_s3_query_string() { + assert_eq!( + with_extension("s3://bucket/base/rsa_key?region=us-west-2", "pem"), + "s3://bucket/base/rsa_key.pem?region=us-west-2" + ); + } + + #[test] + fn splits_s3_parent_and_file_name_without_query_string() { + let path = "s3://bucket/base/config.json?region=us-west-2"; + + assert_eq!(parent(path).as_deref(), Some("s3://bucket/base?region=us-west-2")); + assert_eq!(file_name(path).as_deref(), Some("config.json")); + } +}