diff --git a/Cargo.lock b/Cargo.lock index 4b020ed8..719cf32a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -280,9 +280,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.72" +version = "0.3.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17c6a35df3749d2e8bb1b7b21a976d82b15548788d2735b9d82f329268f71a11" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" dependencies = [ "addr2line", "cc", @@ -400,9 +400,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.98" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" +checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" dependencies = [ "jobserver", "libc", @@ -443,9 +443,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.4" +version = "4.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" +checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f" dependencies = [ "clap_builder", "clap_derive", @@ -453,9 +453,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.2" +version = "4.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f" dependencies = [ "anstyle", "clap_lex", @@ -463,9 +463,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.4" +version = "4.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -475,9 +475,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" [[package]] name = "color_quant" @@ -542,6 +542,7 @@ dependencies = [ "tracing-opentelemetry", "tracing-subscriber", "url", + "webpage", ] [[package]] @@ -727,6 +728,17 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "displaydoc" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "ed25519" version = "2.2.3" @@ -852,6 +864,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + [[package]] name = "futures-channel" version = "0.3.30" @@ -1136,6 +1158,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "html5ever" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13771afe0e6e846f1e67d038d4cb29998a6779f93c809212e4e9c32efd244d4" +dependencies = [ + "log", + "mac", + "markup5ever", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "http" version = "0.2.12" @@ -1190,12 +1226,12 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", - "futures-core", + "futures-util", "http 1.1.0", "http-body 1.0.0", "pin-project-lite", @@ -1307,6 +1343,124 @@ dependencies = [ "tracing", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "idna" version = "0.4.0" @@ -1319,12 +1473,14 @@ dependencies = [ [[package]] name = "idna" -version = "0.5.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "icu_normalizer", + "icu_properties", + "smallvec", + "utf8_iter", ] [[package]] @@ -1540,6 +1696,12 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "lock_api" version = "0.4.12" @@ -1567,20 +1729,52 @@ dependencies = [ [[package]] name = "lz4-sys" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +checksum = "e9764018d143cc854c9f17f0b907de70f14393b1f502da6375dce70f00514eb3" dependencies = [ "cc", "libc", ] +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + [[package]] name = "maplit" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" +[[package]] +name = "markup5ever" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16ce3abbeba692c8b8441d036ef91aea6df8da2c6b6e21c7e14d3c18e526be45" +dependencies = [ + "log", + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", + "tendril", +] + +[[package]] +name = "markup5ever_rcdom" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edaa21ab3701bfee5099ade5f7e1f84553fd19228cf332f13cd6e964bf59be18" +dependencies = [ + "html5ever", + "markup5ever", + "tendril", + "xml5ever", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -1604,9 +1798,9 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "mime" @@ -1641,6 +1835,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "nix" version = "0.28.0" @@ -1719,9 +1919,9 @@ dependencies = [ [[package]] name = "object" -version = "0.35.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8ec7ab813848ba4522158d5517a6093db1ded27575b070f4177b8d12b41db5e" +checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" dependencies = [ "memchr", ] @@ -1920,6 +2120,63 @@ dependencies = [ "zigzag", ] +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_shared 0.11.2", +] + +[[package]] +name = "phf_codegen" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +dependencies = [ + "phf_generator 0.11.2", + "phf_shared 0.11.2", +] + +[[package]] +name = "phf_generator" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" +dependencies = [ + "phf_shared 0.10.0", + "rand", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared 0.11.2", + "rand", +] + +[[package]] +name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.5" @@ -1999,6 +2256,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "proc-macro-crate" version = "3.1.0" @@ -2100,9 +2363,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" dependencies = [ "bitflags 2.5.0", ] @@ -2120,14 +2383,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.4" +version = "1.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.6", - "regex-syntax 0.8.3", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", ] [[package]] @@ -2141,13 +2404,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.3", + "regex-syntax 0.8.4", ] [[package]] @@ -2158,9 +2421,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "reqwest" @@ -2839,6 +3102,12 @@ dependencies = [ "time", ] +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + [[package]] name = "slab" version = "0.4.9" @@ -2880,6 +3149,38 @@ dependencies = [ "der", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "string_cache" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" +dependencies = [ + "new_debug_unreachable", + "once_cell", + "parking_lot", + "phf_shared 0.10.0", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" +dependencies = [ + "phf_generator 0.10.0", + "phf_shared 0.10.0", + "proc-macro2", + "quote", +] + [[package]] name = "subslice" version = "0.2.3" @@ -2918,6 +3219,28 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + [[package]] name = "thiserror" version = "1.0.61" @@ -3008,6 +3331,16 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -3161,7 +3494,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.11", + "winnow 0.6.13", ] [[package]] @@ -3409,12 +3742,12 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56" dependencies = [ "form_urlencoded", - "idna 0.5.0", + "idna 1.0.0", "percent-encoding", "serde", ] @@ -3425,6 +3758,24 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "uuid" version = "1.8.0" @@ -3553,6 +3904,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpage" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70862efc041d46e6bbaa82bb9c34ae0596d090e86cbd14bd9e93b36ee6802eac" +dependencies = [ + "html5ever", + "markup5ever_rcdom", + "serde_json", + "url", +] + [[package]] name = "weezl" version = "0.1.8" @@ -3743,9 +4106,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.11" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c52728401e1dc672a56e81e593e912aa54c78f40246869f78359a2bf24d29d" +checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" dependencies = [ "memchr", ] @@ -3770,12 +4133,59 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "xml5ever" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bbb26405d8e919bc1547a5aa9abc95cbfa438f04844f5fdd9dc7596b748bf69" +dependencies = [ + "log", + "mac", + "markup5ever", +] + [[package]] name = "yansi" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.34" @@ -3796,12 +4206,55 @@ dependencies = [ "syn", ] +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerovec" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zigzag" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 0cdde4ab..d440acce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -147,6 +147,8 @@ tikv-jemallocator = { version = "0.5.0", features = [ sd-notify = { version = "0.4.1", optional = true } +webpage = { version = "2.0", default-features = false } + # Used for matrix spec type definitions and helpers [dependencies.ruma] features = [ diff --git a/conduit-example.toml b/conduit-example.toml index 74cbb074..f35a3d63 100644 --- a/conduit-example.toml +++ b/conduit-example.toml @@ -66,6 +66,8 @@ trusted_servers = ["matrix.org"] address = "127.0.0.1" # This makes sure Conduit can only be reached using the reverse proxy #address = "0.0.0.0" # If Conduit is running in a container, make sure the reverse proxy (ie. Traefik) can reach it. +url_preview_allowlist = [] + [global.well_known] # Conduit handles the /.well-known/matrix/* endpoints, making both clients and servers try to access conduit with the host # server_name and port 443 by default. diff --git a/debian/postinst b/debian/postinst index 6361af5a..cf8937ff 100644 --- a/debian/postinst +++ b/debian/postinst @@ -96,6 +96,8 @@ trusted_servers = ["matrix.org"] # # [0]: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives #log = "..." + +url_preview_allowlist = [] EOF fi ;; diff --git a/src/api/client_server/media.rs b/src/api/client_server/media.rs index 03e4cbab..48b4f56a 100644 --- a/src/api/client_server/media.rs +++ b/src/api/client_server/media.rs @@ -3,7 +3,12 @@ use std::time::Duration; -use crate::{service::media::FileMeta, services, utils, Error, Result, Ruma}; +use crate::{ + config::UrlPreviewPermission, + service::media::{FileMeta, UrlPreviewData}, + services, utils, Error, Result, Ruma, +}; +use hickory_resolver::error::ResolveErrorKind; use http::header::{CONTENT_DISPOSITION, CONTENT_TYPE}; use ruma::{ api::{ @@ -12,7 +17,7 @@ use ruma::{ get_content, get_content_as_filename, get_content_thumbnail, get_media_config, }, error::ErrorKind, - media::{self, create_content}, + media::{self, create_content, get_media_preview}, }, federation::authenticated_media::{self as federation_media, FileOrLocation}, }, @@ -21,6 +26,11 @@ use ruma::{ ServerName, UInt, }; +use image::io::Reader as ImgReader; +use reqwest::Url; +use std::{io::Cursor, net::IpAddr, sync::Arc}; +use webpage::HTML; + const MXC_LENGTH: usize = 32; /// # `GET /_matrix/media/r0/config` @@ -45,6 +55,282 @@ pub async fn get_media_config_auth_route( }) } +async fn download_image(client: &reqwest::Client, url: &str) -> Result { + let image = client.get(url).send().await?.bytes().await?; + let mxc = format!( + "mxc://{}/{}", + services().globals.server_name(), + utils::random_string(MXC_LENGTH) + ); + services() + .media + .create(mxc.clone(), None, None, &image) + .await?; + + let (width, height) = match ImgReader::new(Cursor::new(&image)).with_guessed_format() { + Err(_) => (None, None), + Ok(reader) => match reader.into_dimensions() { + Err(_) => (None, None), + Ok((width, height)) => (Some(width), Some(height)), + }, + }; + + Ok(UrlPreviewData { + image: mxc, + image_size: Some(image.len()), + image_width: width, + image_height: height, + ..Default::default() + }) +} + +async fn download_html(client: &reqwest::Client, url: &str) -> Result { + let max_download_size = 300_000; + + let mut response = client.get(url).send().await?; + + let mut bytes: Vec = Vec::new(); + while let Some(chunk) = response.chunk().await? { + bytes.extend_from_slice(&chunk); + if bytes.len() > max_download_size { + break; + } + } + let body = String::from_utf8_lossy(&bytes); + let html = match HTML::from_string(body.to_string(), Some(url.to_owned())) { + Ok(html) => html, + Err(_) => { + return Err(Error::BadRequest( + ErrorKind::Unknown, + "Failed to parse HTML", + )) + } + }; + + let mut data = match html.opengraph.images.first() { + None => UrlPreviewData::default(), + Some(obj) => download_image(client, &obj.url).await?, + }; + + let props = html.opengraph.properties; + /* use OpenGraph title/description, but fall back to HTML if not available */ + data.title = props + .get("title") + .cloned() + .or(html.title) + .unwrap_or(String::from(url)); + data.description = props.get("description").cloned().or(html.description); + Ok(data) +} + +fn is_ip_external(addr: &IpAddr) -> bool { + // could be implemented with reqwest when it supports IP filtering: + // https://github.com/seanmonstar/reqwest/issues/1515 + + // These checks have been taken from the Rust core/net/ipaddr.rs crate, + // IpAddr::V4.is_global() and IpAddr::V6.is_global(), as .is_global is not + // yet stabilized. TODO: Once this is stable, this match can be simplified. + match addr { + IpAddr::V4(ip4) => { + !(ip4.octets()[0] == 0 // "This network" + || ip4.is_private() + || (ip4.octets()[0] == 100 && (ip4.octets()[1] & 0b1100_0000 == 0b0100_0000)) // is_shared() + || ip4.is_loopback() + || ip4.is_link_local() + // addresses reserved for future protocols (`192.0.0.0/24`) + || (ip4.octets()[0] == 192 && ip4.octets()[1] == 0 && ip4.octets()[2] == 0) + || ip4.is_documentation() + || (ip4.octets()[0] == 198 && (ip4.octets()[1] & 0xfe) == 18) // is_benchmarking() + || (ip4.octets()[0] & 240 == 240 && !ip4.is_broadcast()) // is_reserved() + || ip4.is_broadcast()) + } + IpAddr::V6(ip6) => { + !(ip6.is_unspecified() + || ip6.is_loopback() + // IPv4-mapped Address (`::ffff:0:0/96`) + || matches!(ip6.segments(), [0, 0, 0, 0, 0, 0xffff, _, _]) + // IPv4-IPv6 Translat. (`64:ff9b:1::/48`) + || matches!(ip6.segments(), [0x64, 0xff9b, 1, _, _, _, _, _]) + // Discard-Only Address Block (`100::/64`) + || matches!(ip6.segments(), [0x100, 0, 0, 0, _, _, _, _]) + // IETF Protocol Assignments (`2001::/23`) + || (matches!(ip6.segments(), [0x2001, b, _, _, _, _, _, _] if b < 0x200) + && !( + // Port Control Protocol Anycast (`2001:1::1`) + u128::from_be_bytes(ip6.octets()) == 0x2001_0001_0000_0000_0000_0000_0000_0001 + // Traversal Using Relays around NAT Anycast (`2001:1::2`) + || u128::from_be_bytes(ip6.octets()) == 0x2001_0001_0000_0000_0000_0000_0000_0002 + // AMT (`2001:3::/32`) + || matches!(ip6.segments(), [0x2001, 3, _, _, _, _, _, _]) + // AS112-v6 (`2001:4:112::/48`) + || matches!(ip6.segments(), [0x2001, 4, 0x112, _, _, _, _, _]) + // ORCHIDv2 (`2001:20::/28`) + || matches!(ip6.segments(), [0x2001, b, _, _, _, _, _, _] if (0x20..=0x2f).contains(&b)) + )) + || ((ip6.segments()[0] == 0x2001) && (ip6.segments()[1] == 0xdb8)) // is_documentation() + || ((ip6.segments()[0] & 0xfe00) == 0xfc00) // is_unique_local() + || ((ip6.segments()[0] & 0xffc0) == 0xfe80)) // is_unicast_link_local + } + } +} + +/// Generate URL preview data from the given URL +async fn request_url_preview(url: &Url) -> Result { + // host guaranteed to not be None by get_media_preview_route + let host = url.host_str().unwrap(); + + // resolve host to IP to ensure it's not an internal IP + let dns_resolver = services().globals.dns_resolver(); + match dns_resolver.lookup_ip(format!("{host}.")).await { + Err(_) => { + return Err(Error::BadServerResponse( + "Failed to resolve media preview host", + )); + } + Ok(lookup) if lookup.iter().any(|ip| !is_ip_external(&ip)) => { + return Err(Error::BadRequest( + ErrorKind::Unknown, + "Requesting from this address forbidden", + )); + } + Ok(_) => {} + } + + // Spamhaus API is over DNS. Query the API domain, no result = no block + // https://docs.spamhaus.com/datasets/docs/source/70-access-methods/data-query-service/040-dqs-queries.html + if services().globals.url_previews().use_spamhaus_denylist { + let resolver = services().globals.dns_resolver(); + match resolver + .lookup_ip(format!("{host}.dbl.spamhaus.org.")) + .await + { + Err(e) => { + if let ResolveErrorKind::NoRecordsFound { .. } = e.kind() { + } else { + tracing::log::warn!("Failed to check Spamhaus denylist: {}", e); + } + } + Ok(_) => { + return Err(Error::BadRequest( + ErrorKind::Unknown, + "Domain fails reputation check", + )); + } + } + } + + let client = services().globals.default_client(); + let response = client.head(url.as_str()).send().await?; + + let content_type = match response + .headers() + .get(CONTENT_TYPE) + .and_then(|x| x.to_str().ok()) + { + Some(ct) => ct, + None => { + return Err(Error::BadRequest( + ErrorKind::Unknown, + "Unknown Content-Type", + )) + } + }; + let data = match content_type { + html if html.starts_with("text/html") => download_html(&client, url.as_str()).await?, + img if img.starts_with("image/") => download_image(&client, url.as_str()).await?, + _ => { + return Err(Error::BadRequest( + ErrorKind::Unknown, + "Unsupported Content-Type", + )) + } + }; + + services() + .media + .set_url_preview(url.as_str(), &data) + .await?; + + Ok(data) +} + +/// Retrieve URL preview data from database if available, or generate it +async fn get_url_preview(url: &Url) -> Result { + if let Some(preview) = services().media.get_url_preview(url.as_str()).await { + return Ok(preview); + } + + // ensure that only one request is made per URL + let mutex_request = Arc::clone( + services() + .media + .url_preview_mutex + .write() + .unwrap() + .entry(url.as_str().to_owned()) + .or_default(), + ); + let _request_lock = mutex_request.lock().await; + + match services().media.get_url_preview(url.as_str()).await { + Some(preview) => Ok(preview), + None => request_url_preview(url).await, + } +} + +/// Verify that the given URL's host is in the allow list. +fn url_preview_allowed(url: &Url) -> bool { + // host's existence is already verified in get_media_preview_route, unwrap is safe + let host = url.host_str().unwrap().to_lowercase(); + let preview_config = services().globals.url_previews(); + match preview_config.default { + UrlPreviewPermission::Forbid => { + preview_config.exceptions.iter().any(|ex| ex.matches(&host)) + } + UrlPreviewPermission::Allow => { + !preview_config.exceptions.iter().any(|ex| ex.matches(&host)) + } + } +} + +/// # `GET /_matrix/media/r0/preview_url` +/// +/// Returns URL preview. +pub async fn get_media_preview_route( + body: Ruma, +) -> Result { + let url = match Url::parse(&body.url) { + Err(_) => { + return Err(Error::BadRequest(ErrorKind::Unknown, "Not a valid URL")); + } + Ok(u) if u.scheme() != "http" && u.scheme() != "https" || u.host().is_none() => { + return Err(Error::BadRequest( + ErrorKind::Unknown, + "Not a valid HTTP URL", + )); + } + Ok(url) => url, + }; + + if !url_preview_allowed(&url) { + return Err(Error::BadRequest( + ErrorKind::Unknown, + "Previewing URL not allowed", + )); + } + + match get_url_preview(&url).await { + Ok(preview) => { + let res = serde_json::value::to_raw_value(&preview).expect("Converting to JSON failed"); + Ok(get_media_preview::v3::Response::from_raw_value(res)) + } + Err(_) => Err(Error::BadRequest( + ErrorKind::NotFound, + "Failed to find preview data", + )), + } +} + /// # `POST /_matrix/media/r0/upload` /// /// Permanently save media in the server. diff --git a/src/config/mod.rs b/src/config/mod.rs index 378ab929..20c9c241 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -8,8 +8,10 @@ use ruma::{OwnedServerName, RoomVersionId}; use serde::{de::IgnoredAny, Deserialize}; use tracing::warn; use url::Url; +use wild_carded_domain::WildCardedDomain; mod proxy; +mod wild_carded_domain; use self::proxy::ProxyConfig; @@ -85,6 +87,9 @@ pub struct Config { pub emergency_password: Option, + #[serde(default)] + pub url_previews: UrlPreviewConfig, + #[serde(flatten)] pub catchall: BTreeMap, } @@ -101,6 +106,38 @@ pub struct WellKnownConfig { pub server: Option, } +#[derive(Clone, Debug, Deserialize, Default)] +pub struct UrlPreviewConfig { + pub default: UrlPreviewPermission, + #[serde(default)] + pub exceptions: Vec, + #[serde(default)] + pub use_spamhaus_denylist: bool, +} + +#[derive(Clone, Debug, Deserialize, Default)] +pub enum UrlPreviewPermission { + Allow, + #[default] + Forbid, +} +impl UrlPreviewPermission { + pub fn invert(&self) -> Self { + match self { + UrlPreviewPermission::Allow => UrlPreviewPermission::Forbid, + UrlPreviewPermission::Forbid => UrlPreviewPermission::Allow, + } + } +} +impl fmt::Display for UrlPreviewPermission { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + UrlPreviewPermission::Allow => write!(f, "ALLOW"), + UrlPreviewPermission::Forbid => write!(f, "FORBID"), + } + } +} + const DEPRECATED_KEYS: &[&str] = &["cache_capacity"]; impl Config { @@ -232,6 +269,14 @@ impl fmt::Display for Config { }), ("Well-known server name", well_known_server.as_str()), ("Well-known client URL", &self.well_known_client()), + ("URL preview", { + let mut lst = vec![]; + for exc in &self.url_previews.exceptions { + lst.push(format!("{} {}", self.url_previews.default.invert(), exc)); + } + lst.push(format!("{} {}", self.url_previews.default, "*")); + &lst.join(", ") + }), ]; let mut msg: String = "Active config values:\n\n".to_owned(); diff --git a/src/config/proxy.rs b/src/config/proxy.rs index c03463e7..0b604647 100644 --- a/src/config/proxy.rs +++ b/src/config/proxy.rs @@ -1,6 +1,7 @@ use reqwest::{Proxy, Url}; use serde::Deserialize; +use super::wild_carded_domain::WildCardedDomain; use crate::Result; /// ## Examples: @@ -92,52 +93,3 @@ impl PartialProxyConfig { } } } - -/// A domain name, that optionally allows a * as its first subdomain. -#[derive(Clone, Debug)] -pub enum WildCardedDomain { - WildCard, - WildCarded(String), - Exact(String), -} -impl WildCardedDomain { - pub fn matches(&self, domain: &str) -> bool { - match self { - WildCardedDomain::WildCard => true, - WildCardedDomain::WildCarded(d) => domain.ends_with(d), - WildCardedDomain::Exact(d) => domain == d, - } - } - pub fn more_specific_than(&self, other: &Self) -> bool { - match (self, other) { - (WildCardedDomain::WildCard, WildCardedDomain::WildCard) => false, - (_, WildCardedDomain::WildCard) => true, - (WildCardedDomain::Exact(a), WildCardedDomain::WildCarded(_)) => other.matches(a), - (WildCardedDomain::WildCarded(a), WildCardedDomain::WildCarded(b)) => { - a != b && a.ends_with(b) - } - _ => false, - } - } -} -impl std::str::FromStr for WildCardedDomain { - type Err = std::convert::Infallible; - fn from_str(s: &str) -> Result { - // maybe do some domain validation? - Ok(if s.starts_with("*.") { - WildCardedDomain::WildCarded(s[1..].to_owned()) - } else if s == "*" { - WildCardedDomain::WildCarded("".to_owned()) - } else { - WildCardedDomain::Exact(s.to_owned()) - }) - } -} -impl<'de> Deserialize<'de> for WildCardedDomain { - fn deserialize(deserializer: D) -> Result - where - D: serde::de::Deserializer<'de>, - { - crate::utils::deserialize_from_str(deserializer) - } -} diff --git a/src/config/wild_carded_domain.rs b/src/config/wild_carded_domain.rs new file mode 100644 index 00000000..9452f230 --- /dev/null +++ b/src/config/wild_carded_domain.rs @@ -0,0 +1,64 @@ +use serde::Deserialize; +use std::fmt; + +/// A domain name, that optionally allows a * as its first subdomain. +#[derive(Clone, Debug)] +pub enum WildCardedDomain { + WildCard, + WildCarded(String), + Exact(String), +} + +impl WildCardedDomain { + pub fn matches(&self, domain: &str) -> bool { + match self { + WildCardedDomain::WildCard => true, + WildCardedDomain::WildCarded(d) => domain.ends_with(d), + WildCardedDomain::Exact(d) => domain == d, + } + } + pub fn more_specific_than(&self, other: &Self) -> bool { + match (self, other) { + (WildCardedDomain::WildCard, WildCardedDomain::WildCard) => false, + (_, WildCardedDomain::WildCard) => true, + (WildCardedDomain::Exact(a), WildCardedDomain::WildCarded(_)) => other.matches(a), + (WildCardedDomain::WildCarded(a), WildCardedDomain::WildCarded(b)) => { + a != b && a.ends_with(b) + } + _ => false, + } + } +} + +impl std::str::FromStr for WildCardedDomain { + type Err = std::convert::Infallible; + fn from_str(s: &str) -> Result { + // maybe do some domain validation? + Ok(if s.starts_with("*.") { + WildCardedDomain::WildCarded(s[1..].to_lowercase()) + } else if s == "*" { + WildCardedDomain::WildCarded("".to_lowercase()) + } else { + WildCardedDomain::Exact(s.to_lowercase()) + }) + } +} + +impl<'de> Deserialize<'de> for WildCardedDomain { + fn deserialize(deserializer: D) -> Result + where + D: serde::de::Deserializer<'de>, + { + crate::utils::deserialize_from_str(deserializer) + } +} + +impl fmt::Display for WildCardedDomain { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + WildCardedDomain::WildCard => write!(f, "*"), + WildCardedDomain::WildCarded(d) => write!(f, "*{d}"), + WildCardedDomain::Exact(d) => write!(f, "{d}"), + } + } +} diff --git a/src/database/key_value/media.rs b/src/database/key_value/media.rs index 99df0097..f229633d 100644 --- a/src/database/key_value/media.rs +++ b/src/database/key_value/media.rs @@ -1,6 +1,10 @@ use ruma::{api::client::error::ErrorKind, http_headers::ContentDisposition}; -use crate::{database::KeyValueDatabase, service, utils, Error, Result}; +use crate::{ + database::KeyValueDatabase, + service::{self, media::UrlPreviewData}, + utils, Error, Result, +}; impl service::media::Data for KeyValueDatabase { fn create_file_metadata( @@ -68,4 +72,96 @@ impl service::media::Data for KeyValueDatabase { }); Ok((content_disposition, content_type, key)) } + + fn remove_url_preview(&self, url: &str) -> Result<()> { + self.url_previews.remove(url.as_bytes()) + } + + fn set_url_preview( + &self, + url: &str, + data: &UrlPreviewData, + timestamp: std::time::Duration, + ) -> Result<()> { + let mut value = Vec::::new(); + value.extend_from_slice(×tamp.as_secs().to_be_bytes()); + value.push(0xff); + value.extend_from_slice(data.title.as_bytes()); + value.push(0xff); + value.extend_from_slice( + data.description + .as_ref() + .map(|d| d.as_bytes()) + .unwrap_or_default(), + ); + value.push(0xff); + value.extend_from_slice(data.image.as_bytes()); + value.push(0xff); + value.extend_from_slice(&data.image_size.unwrap_or(0).to_be_bytes()); + value.push(0xff); + value.extend_from_slice(&data.image_width.unwrap_or(0).to_be_bytes()); + value.push(0xff); + value.extend_from_slice(&data.image_height.unwrap_or(0).to_be_bytes()); + + self.url_previews.insert(url.as_bytes(), &value) + } + + fn get_url_preview(&self, url: &str) -> Option { + let values = self.url_previews.get(url.as_bytes()).ok()??; + + let mut values = values.split(|&b| b == 0xff); + + let _ts = match values + .next() + .map(|b| u64::from_be_bytes(b.try_into().expect("valid BE array"))) + { + Some(0) => None, + x => x, + }; + let title = values + .next() + .and_then(|b| String::from_utf8(b.to_vec()).ok()) + .unwrap_or_default(); + let description = match values + .next() + .and_then(|b| String::from_utf8(b.to_vec()).ok()) + { + Some(s) if s.is_empty() => None, + x => x, + }; + let image = values + .next() + .and_then(|b| String::from_utf8(b.to_vec()).ok()) + .unwrap_or_default(); + let image_size = match values + .next() + .map(|b| usize::from_be_bytes(b.try_into().expect("valid BE array"))) + { + Some(0) => None, + x => x, + }; + let image_width = match values + .next() + .map(|b| u32::from_be_bytes(b.try_into().expect("valid BE array"))) + { + Some(0) => None, + x => x, + }; + let image_height = match values + .next() + .map(|b| u32::from_be_bytes(b.try_into().expect("valid BE array"))) + { + Some(0) => None, + x => x, + }; + + Some(UrlPreviewData { + title, + description, + image, + image_size, + image_width, + image_height, + }) + } } diff --git a/src/database/mod.rs b/src/database/mod.rs index 2317f7a8..83632d47 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -147,6 +147,7 @@ pub struct KeyValueDatabase { //pub media: media::Media, pub(super) mediaid_file: Arc, // MediaId = MXC + WidthHeight + ContentDisposition + ContentType + pub(super) url_previews: Arc, //pub key_backups: key_backups::KeyBackups, pub(super) backupid_algorithm: Arc, // BackupId = UserId + Version(Count) pub(super) backupid_etag: Arc, // BackupId = UserId + Version(Count) @@ -363,6 +364,7 @@ impl KeyValueDatabase { roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?, roomusertype_roomuserdataid: builder.open_tree("roomusertype_roomuserdataid")?, mediaid_file: builder.open_tree("mediaid_file")?, + url_previews: builder.open_tree("url_previews")?, backupid_algorithm: builder.open_tree("backupid_algorithm")?, backupid_etag: builder.open_tree("backupid_etag")?, backupkeyid_backup: builder.open_tree("backupkeyid_backup")?, diff --git a/src/main.rs b/src/main.rs index 2776c200..831f3896 100644 --- a/src/main.rs +++ b/src/main.rs @@ -396,6 +396,7 @@ fn routes(config: &Config) -> Router { .ruma_route(client_server::send_event_to_device_route) .ruma_route(client_server::get_media_config_route) .ruma_route(client_server::get_media_config_auth_route) + .ruma_route(client_server::get_media_preview_route) .ruma_route(client_server::create_content_route) .ruma_route(client_server::get_content_route) .ruma_route(client_server::get_content_auth_route) diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 3325e518..3048d076 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -7,7 +7,7 @@ use ruma::{ use crate::api::server_server::DestinationResponse; -use crate::{services, Config, Error, Result}; +use crate::{config::UrlPreviewConfig, services, Config, Error, Result}; use futures_util::FutureExt; use hickory_resolver::TokioAsyncResolver; use hyper_util::client::legacy::connect::dns::{GaiResolver, Name as HyperName}; @@ -324,6 +324,10 @@ impl Service { self.config.allow_federation } + pub fn url_previews(&self) -> &UrlPreviewConfig { + &self.config.url_previews + } + pub fn allow_room_creation(&self) -> bool { self.config.allow_room_creation } diff --git a/src/service/media/data.rs b/src/service/media/data.rs index 844aa995..97da0a26 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -19,4 +19,15 @@ pub trait Data: Send + Sync { width: u32, height: u32, ) -> Result<(ContentDisposition, Option, Vec)>; + + fn remove_url_preview(&self, url: &str) -> Result<()>; + + fn set_url_preview( + &self, + url: &str, + data: &super::UrlPreviewData, + timestamp: std::time::Duration, + ) -> Result<()>; + + fn get_url_preview(&self, url: &str) -> Option; } diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index a7ac9d50..33d64446 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -1,5 +1,10 @@ mod data; -use std::io::Cursor; +use std::{ + collections::HashMap, + io::Cursor, + sync::{Arc, RwLock}, + time::SystemTime, +}; pub use data::Data; use ruma::http_headers::{ContentDisposition, ContentDispositionType}; @@ -7,9 +12,11 @@ use ruma::http_headers::{ContentDisposition, ContentDispositionType}; use crate::{services, Result}; use image::imageops::FilterType; +use serde::Serialize; use tokio::{ fs::File, io::{AsyncReadExt, AsyncWriteExt, BufReader}, + sync::Mutex, }; pub struct FileMeta { @@ -18,8 +25,37 @@ pub struct FileMeta { pub file: Vec, } +#[derive(Serialize, Default)] +pub struct UrlPreviewData { + #[serde(rename(serialize = "og:title"))] + pub title: String, + #[serde( + skip_serializing_if = "Option::is_none", + rename(serialize = "og:description") + )] + pub description: Option, + #[serde(rename(serialize = "og:image"))] + pub image: String, + #[serde( + skip_serializing_if = "Option::is_none", + rename(serialize = "matrix:image:size") + )] + pub image_size: Option, + #[serde( + skip_serializing_if = "Option::is_none", + rename(serialize = "og:image:width") + )] + pub image_width: Option, + #[serde( + skip_serializing_if = "Option::is_none", + rename(serialize = "og:image:height") + )] + pub image_height: Option, +} + pub struct Service { pub db: &'static dyn Data, + pub url_preview_mutex: RwLock>>>, } impl Service { @@ -230,4 +266,20 @@ impl Service { Ok(None) } } + + pub async fn get_url_preview(&self, url: &str) -> Option { + self.db.get_url_preview(url) + } + + pub async fn remove_url_preview(&self, url: &str) -> Result<()> { + // TODO: also remove the downloaded image + self.db.remove_url_preview(url) + } + + pub async fn set_url_preview(&self, url: &str, data: &UrlPreviewData) -> Result<()> { + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("valid system time"); + self.db.set_url_preview(url, data, now) + } } diff --git a/src/service/mod.rs b/src/service/mod.rs index 552c71af..cb418254 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1,6 +1,6 @@ use std::{ collections::{BTreeMap, HashMap}, - sync::{Arc, Mutex as StdMutex}, + sync::{Arc, Mutex as StdMutex, RwLock as StdRwLock}, }; use lru_cache::LruCache; @@ -118,7 +118,10 @@ impl Services { account_data: account_data::Service { db }, admin: admin::Service::build(), key_backups: key_backups::Service { db }, - media: media::Service { db }, + media: media::Service { + db, + url_preview_mutex: StdRwLock::new(HashMap::new()), + }, sending: sending::Service::build(db, &config), globals: globals::Service::load(db, config)?, diff --git a/tests/test-config.toml b/tests/test-config.toml index 10db1408..c6773a16 100644 --- a/tests/test-config.toml +++ b/tests/test-config.toml @@ -8,6 +8,7 @@ database_path = "/tmp" # All the other settings are left at their defaults: address = "127.0.0.1" +database_backend = "rocksdb" allow_registration = true max_request_size = 20_000_000 port = 6167