mirror of
https://gitlab.com/famedly/conduit.git
synced 2025-06-27 16:35:59 +00:00
Merge branch 'media-refactor' into 'next'
Media refactor Closes #436, #146, #312, #168, #421, and #496 See merge request famedly/conduit!740
This commit is contained in:
Matthias Ahouansou
commit
215198d1c0
28 changed files with 4607 additions and 589 deletions
|
|
@ -26,7 +26,7 @@ before_script:
|
|||
|
||||
# Add our own binary cache
|
||||
- if command -v nix > /dev/null; then echo "extra-substituters = https://attic.conduit.rs/conduit" >> /etc/nix/nix.conf; fi
|
||||
- if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduit:ddcaWZiWm0l0IXZlO8FERRdWvEufwmd0Negl1P+c0Ns=" >> /etc/nix/nix.conf; fi
|
||||
- if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduit:zXpsVmtm+MBbweaCaG/CT4pCEDDjfFAKjgbCqfDBjLE=" >> /etc/nix/nix.conf; fi
|
||||
|
||||
# Add alternate binary cache
|
||||
- if command -v nix > /dev/null && [ -n "$ATTIC_ENDPOINT" ]; then echo "extra-substituters = $ATTIC_ENDPOINT" >> /etc/nix/nix.conf; fi
|
||||
|
|
@ -84,10 +84,10 @@ artifacts:
|
|||
- ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl
|
||||
- cp result/bin/conduit x86_64-unknown-linux-musl
|
||||
|
||||
- mkdir -p target/release
|
||||
- cp result/bin/conduit target/release
|
||||
- direnv exec . cargo deb --no-build
|
||||
- mv target/debian/*.deb x86_64-unknown-linux-musl.deb
|
||||
- mkdir -p target/x86_64-unknown-linux-musl/release
|
||||
- cp result/bin/conduit target/x86_64-unknown-linux-musl/release
|
||||
- direnv exec . cargo deb --no-build --target x86_64-unknown-linux-musl
|
||||
- mv target/x86_64-unknown-linux-musl/debian/*.deb x86_64-unknown-linux-musl.deb
|
||||
|
||||
# Since the OCI image package is based on the binary package, this has the
|
||||
# fun side effect of uploading the normal binary too. Conduit users who are
|
||||
|
|
@ -105,6 +105,7 @@ artifacts:
|
|||
|
||||
- mkdir -p target/aarch64-unknown-linux-musl/release
|
||||
- cp result/bin/conduit target/aarch64-unknown-linux-musl/release
|
||||
# binary stripping requires target-specific binary (`strip`)
|
||||
- direnv exec . cargo deb --no-strip --no-build --target aarch64-unknown-linux-musl
|
||||
- mv target/aarch64-unknown-linux-musl/debian/*.deb aarch64-unknown-linux-musl.deb
|
||||
|
||||
|
|
|
|||
132
Cargo.lock
generated
132
Cargo.lock
generated
|
|
@ -1,6 +1,6 @@
|
|||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 3
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "addr2line"
|
||||
|
|
@ -38,6 +38,21 @@ dependencies = [
|
|||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "android-tzdata"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
|
||||
|
||||
[[package]]
|
||||
name = "android_system_properties"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.7"
|
||||
|
|
@ -381,12 +396,27 @@ version = "1.5.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder-lite"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
|
||||
|
||||
[[package]]
|
||||
name = "bytesize"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a3c8f83209414aacf0eeae3cf730b18d6981697fba62f200fcfb92b9f082acba"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bzip2-sys"
|
||||
version = "0.1.11+1.0.8"
|
||||
|
|
@ -430,6 +460,20 @@ version = "0.1.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.40"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c"
|
||||
dependencies = [
|
||||
"android-tzdata",
|
||||
"iana-time-zone",
|
||||
"js-sys",
|
||||
"num-traits",
|
||||
"wasm-bindgen",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clang-sys"
|
||||
version = "1.8.1"
|
||||
|
|
@ -495,13 +539,18 @@ dependencies = [
|
|||
"axum-server",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"bytesize",
|
||||
"chrono",
|
||||
"clap",
|
||||
"directories",
|
||||
"figment",
|
||||
"futures-util",
|
||||
"hex",
|
||||
"hickory-resolver",
|
||||
"hmac",
|
||||
"http 1.1.0",
|
||||
"humantime",
|
||||
"humantime-serde",
|
||||
"hyper 1.3.1",
|
||||
"hyper-util",
|
||||
"image",
|
||||
|
|
@ -528,6 +577,7 @@ dependencies = [
|
|||
"serde_json",
|
||||
"serde_yaml",
|
||||
"sha-1",
|
||||
"sha2",
|
||||
"thiserror 1.0.61",
|
||||
"thread_local",
|
||||
"threadpool",
|
||||
|
|
@ -1045,6 +1095,12 @@ version = "0.3.9"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
|
||||
|
||||
[[package]]
|
||||
name = "hex"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
|
||||
|
||||
[[package]]
|
||||
name = "hickory-proto"
|
||||
version = "0.24.1"
|
||||
|
|
@ -1187,6 +1243,22 @@ version = "1.0.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
|
||||
|
||||
[[package]]
|
||||
name = "humantime"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f"
|
||||
|
||||
[[package]]
|
||||
name = "humantime-serde"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c"
|
||||
dependencies = [
|
||||
"humantime",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "0.14.29"
|
||||
|
|
@ -1281,6 +1353,30 @@ dependencies = [
|
|||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.62"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2fd658b06e56721792c5df4475705b6cda790e9298d19d2f8af083457bcd127"
|
||||
dependencies = [
|
||||
"android_system_properties",
|
||||
"core-foundation-sys",
|
||||
"iana-time-zone-haiku",
|
||||
"js-sys",
|
||||
"log",
|
||||
"wasm-bindgen",
|
||||
"windows-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone-haiku"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "0.4.0"
|
||||
|
|
@ -1311,12 +1407,23 @@ dependencies = [
|
|||
"byteorder",
|
||||
"color_quant",
|
||||
"gif",
|
||||
"image-webp",
|
||||
"num-traits",
|
||||
"png",
|
||||
"zune-core",
|
||||
"zune-jpeg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "image-webp"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f79afb8cbee2ef20f59ccd477a218c12a93943d075b492015ecb1bb81f8ee904"
|
||||
dependencies = [
|
||||
"byteorder-lite",
|
||||
"quick-error 2.0.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "1.9.3"
|
||||
|
|
@ -2017,6 +2124,12 @@ version = "1.2.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
|
||||
|
||||
[[package]]
|
||||
name = "quick-error"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.36"
|
||||
|
|
@ -2169,7 +2282,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00"
|
||||
dependencies = [
|
||||
"hostname",
|
||||
"quick-error",
|
||||
"quick-error 1.2.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3543,6 +3656,21 @@ version = "0.4.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.52.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
|
||||
dependencies = [
|
||||
"windows-targets 0.52.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38"
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.48.0"
|
||||
|
|
|
|||
12
Cargo.toml
12
Cargo.toml
|
|
@ -19,7 +19,7 @@ repository = "https://gitlab.com/famedly/conduit"
|
|||
version = "0.10.0-alpha"
|
||||
|
||||
# See also `rust-toolchain.toml`
|
||||
rust-version = "1.81.0"
|
||||
rust-version = "1.83.0"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
|
|
@ -84,7 +84,14 @@ image = { version = "0.25", default-features = false, features = [
|
|||
"gif",
|
||||
"jpeg",
|
||||
"png",
|
||||
"webp",
|
||||
] }
|
||||
# Used for creating media filenames
|
||||
hex = "0.4"
|
||||
sha2 = "0.10"
|
||||
# Used for parsing media retention policies from the config
|
||||
bytesize = { version = "2", features = ["serde"] }
|
||||
humantime-serde = "1"
|
||||
# Used to encode server public key
|
||||
base64 = "0.22"
|
||||
# Used when hashing the state
|
||||
|
|
@ -120,6 +127,7 @@ thread_local = "1.1.7"
|
|||
hmac = "0.12.1"
|
||||
sha-1 = "0.10.1"
|
||||
# used for conduit's CLI and admin room command parsing
|
||||
chrono = "0.4"
|
||||
clap = { version = "4.3.0", default-features = false, features = [
|
||||
"derive",
|
||||
"error-context",
|
||||
|
|
@ -128,6 +136,8 @@ clap = { version = "4.3.0", default-features = false, features = [
|
|||
"string",
|
||||
"usage",
|
||||
] }
|
||||
humantime = "2"
|
||||
|
||||
futures-util = { version = "0.3.28", default-features = false }
|
||||
# Used for reading the configuration from conduit.toml & environment variables
|
||||
figment = { version = "0.10.8", features = ["env", "toml"] }
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@
|
|||
- [Debian](deploying/debian.md)
|
||||
- [Docker](deploying/docker.md)
|
||||
- [NixOS](deploying/nixos.md)
|
||||
- [Administration](administration.md)
|
||||
- [Media](administration/media.md)
|
||||
- [TURN](turn.md)
|
||||
- [Appservices](appservices.md)
|
||||
- [FAQ](faq.md)
|
||||
|
|
|
|||
3
docs/administration.md
Normal file
3
docs/administration.md
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
# Administration
|
||||
|
||||
This chapter describes how to perform tasks you may want to do while running Conduit
|
||||
30
docs/administration/media.md
Normal file
30
docs/administration/media.md
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
# Media
|
||||
|
||||
While running Conduit, you may encounter undesirable media, either from other servers, or from local users.
|
||||
|
||||
## From other servers
|
||||
If the media originated from a different server, which itself is not malicious, it should be enough
|
||||
to use the `purge-media-from-server` command to delete the media from the media backend, and then
|
||||
contact the remote server so that they can deal with the offending user(s).
|
||||
|
||||
If you do not need to media deleted as soon as possible, you can use retention policies to only
|
||||
store remote media for a short period of time, meaning that the media will be automatically deleted
|
||||
after some time. As new media can only be accessed over authenticated endpoints, only local users
|
||||
will be able to access the media via your server, so if you're running a single-user server, you
|
||||
don't need to worry about the media being distributed via your server.
|
||||
|
||||
If you know the media IDs, (which you can find with the `list-media` command), you can use the
|
||||
`block-media` to prevent any of those media IDs (or other media with the same SHA256 hash) from
|
||||
being stored in the media backend in the future.
|
||||
|
||||
If the server itself if malicious, then it should probably be [ACLed](https://spec.matrix.org/v1.14/client-server-api/#server-access-control-lists-acls-for-rooms)
|
||||
in rooms it particpates in. In the future, you'll be able to block the remote server from
|
||||
interacting with your server completely.
|
||||
|
||||
## From local users
|
||||
If the undesirable media originates from your own server, you can purge media uploaded by them
|
||||
using the `purge-media-from-users` command. If you also plan to deactivate the user, you can do so
|
||||
with the `--purge-media` flag on either the `deactivate-user` or `deactivate-all` commands. If
|
||||
they keep making new accounts, you can use the `block-media-from-users` command to prevent media
|
||||
with the same SHA256 hash from being uploaded again, as well as using the `allow-registration`
|
||||
command to temporarily prevent users from creating new accounts.
|
||||
|
|
@ -57,9 +57,102 @@ The `global` section contains the following fields:
|
|||
| `turn_uris` | `array` | The TURN URIs | `[]` |
|
||||
| `turn_secret` | `string` | The TURN secret | `""` |
|
||||
| `turn_ttl` | `integer` | The TURN TTL in seconds | `86400` |
|
||||
| `media` | `table` | See the [media configuration](#media) | See the [media configuration](#media) |
|
||||
| `emergency_password` | `string` | Set a password to login as the `conduit` user in case of emergency | N/A |
|
||||
| `well_known` | `table` | Used for [delegation](delegation.md) | See [delegation](delegation.md) |
|
||||
|
||||
### Media
|
||||
The `media` table is used to configure how media is stored and where. Currently, there is only one available
|
||||
backend, that being `filesystem`. The backend can be set using the `backend` field. Example:
|
||||
```toml
|
||||
[global.media]
|
||||
backend = "filesystem" # the default backend
|
||||
```
|
||||
|
||||
#### Filesystem backend
|
||||
The filesystem backend has the following fields:
|
||||
- `path`: The base directory where all the media files will be stored (defaults to
|
||||
`${database_path}/media`)
|
||||
- `directory_structure`: This is a table, used to configure how files are to be distributed within
|
||||
the media directory. It has the following fields:
|
||||
- `depth`: The number sub-directories that should be created for files (default: `2`)
|
||||
- `length`: How long the name of these sub-directories should be (default: `2`)
|
||||
For example, a file may regularly have the name `98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4`
|
||||
(The SHA256 digest of the file's content). If `depth` and `length` were both set to `2`, this file would be stored
|
||||
at `${path}/98/ea/6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4`. If you want to instead have all
|
||||
media files in the base directory with no sub-directories, just set `directory_structure` to be empty, as follows:
|
||||
```toml
|
||||
[global.media]
|
||||
backend = "filesystem"
|
||||
|
||||
[global.media.directory_structure]
|
||||
```
|
||||
|
||||
##### Example:
|
||||
```toml
|
||||
[global.media]
|
||||
backend = "filesystem"
|
||||
path = "/srv/matrix-media"
|
||||
|
||||
[global.media.directory_structure]
|
||||
depth = 4
|
||||
length = 2
|
||||
```
|
||||
|
||||
#### Retention policies
|
||||
Over time, the amount of media will keep growing, even if they were only accessed once.
|
||||
Retention policies allow for media files to automatically be deleted if they meet certain crietia,
|
||||
allowing disk space to be saved.
|
||||
|
||||
This can be configured via the `retention` field of the media config, which is an array with
|
||||
"scopes" specified
|
||||
- `scope`: specifies what type of media this policy applies to. If unset, all other scopes which
|
||||
you have not configured will use this as a default. Possible values: `"local"`, `"remote"`,
|
||||
`"thumbnail"`
|
||||
- `accessed`: the maximum amount of time since the media was last accessed,
|
||||
in the form specified by [`humantime::parse_duration`](https://docs.rs/humantime/2.2.0/humantime/fn.parse_duration.html)
|
||||
(e.g. `"240h"`, `"1400min"`, `"2months"`, etc.)
|
||||
- `created`: the maximum amount of time since the media was created after, in the same format as
|
||||
`accessed` above.
|
||||
- `space`: the maximum amount of space all of the media in this scope can occupy (if no scope is
|
||||
specified, this becomes the total for **all** media). If the creation/downloading of new media,
|
||||
will cause this to be exceeded, the last accessed media will be deleted repetitively until there
|
||||
is enough space for the new media. The format is specified by [`ByteSize`](https://docs.rs/bytesize/2.0.1/bytesize/index.html)
|
||||
(e.g. `"10000MB"`, `"15GiB"`, `"1.5TB"`, etc.)
|
||||
|
||||
Media needs to meet **all** the specified requirements to be kept, otherwise, it will be deleted.
|
||||
This means that thumbnails have to meet both the `"thumbnail"`, and either `"local"` or `"remote"`
|
||||
requirements in order to be kept.
|
||||
|
||||
If the media does not meet the `accessed` or `created` requirement, they will be deleted during a
|
||||
periodic cleanup, which happens every 1/10th of the period of the shortest retention time, with a
|
||||
maximum frequency of every minute, and a minimum of every 24 hours. For example, if I set my
|
||||
`accessed` time for all media to `"2months"`, but override that to be `"48h"` for thumbnails,
|
||||
the cleanup will happen every 4.8 hours.
|
||||
|
||||
##### Example
|
||||
```toml
|
||||
# Total of 40GB for all media
|
||||
[[global.media.retention]] # Notice the double "[]", due to this being a table item in an array
|
||||
space = "40G"
|
||||
|
||||
# Delete remote media not accessed for 30 days, or older than 90 days
|
||||
[[global.media.retention]]
|
||||
scope = "remote"
|
||||
accessed = "30d"
|
||||
created = "90days" # you can mix and match between the long and short format
|
||||
|
||||
# Delete local media not accessed for 1 year
|
||||
[[global.media.retention]]
|
||||
scope = "local"
|
||||
accessed = "1y"
|
||||
|
||||
# Only store 1GB of thumbnails
|
||||
[[global.media.retention]]
|
||||
scope = "thumbnail"
|
||||
space = "1GB"
|
||||
|
||||
```
|
||||
|
||||
### TLS
|
||||
The `tls` table contains the following fields:
|
||||
|
|
|
|||
151
flake.lock
generated
151
flake.lock
generated
|
|
@ -4,16 +4,17 @@
|
|||
"inputs": {
|
||||
"crane": "crane",
|
||||
"flake-compat": "flake-compat",
|
||||
"flake-utils": "flake-utils",
|
||||
"flake-parts": "flake-parts",
|
||||
"nix-github-actions": "nix-github-actions",
|
||||
"nixpkgs": "nixpkgs",
|
||||
"nixpkgs-stable": "nixpkgs-stable"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1707922053,
|
||||
"narHash": "sha256-wSZjK+rOXn+UQiP1NbdNn5/UW6UcBxjvlqr2wh++MbM=",
|
||||
"lastModified": 1738524606,
|
||||
"narHash": "sha256-hPYEJ4juK3ph7kbjbvv7PlU1D9pAkkhl+pwx8fZY53U=",
|
||||
"owner": "zhaofengli",
|
||||
"repo": "attic",
|
||||
"rev": "6eabc3f02fae3683bffab483e614bebfcd476b21",
|
||||
"rev": "ff8a897d1f4408ebbf4d45fa9049c06b3e1e3f4e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -31,11 +32,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1702918879,
|
||||
"narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=",
|
||||
"lastModified": 1722960479,
|
||||
"narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=",
|
||||
"owner": "ipetkov",
|
||||
"repo": "crane",
|
||||
"rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb",
|
||||
"rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -45,23 +46,18 @@
|
|||
}
|
||||
},
|
||||
"crane_2": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1713721181,
|
||||
"narHash": "sha256-Vz1KRVTzU3ClBfyhOj8gOehZk21q58T1YsXC30V23PU=",
|
||||
"lastModified": 1741481578,
|
||||
"narHash": "sha256-JBTSyJFQdO3V8cgcL08VaBUByEU6P5kXbTJN6R0PFQo=",
|
||||
"owner": "ipetkov",
|
||||
"repo": "crane",
|
||||
"rev": "55f4939ac59ff8f89c6a4029730a2d49ea09105f",
|
||||
"rev": "bb1c9567c43e4434f54e9481eb4b8e8e0d50f0b5",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "ipetkov",
|
||||
"ref": "master",
|
||||
"repo": "crane",
|
||||
"rev": "bb1c9567c43e4434f54e9481eb4b8e8e0d50f0b5",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
|
|
@ -73,11 +69,11 @@
|
|||
"rust-analyzer-src": "rust-analyzer-src"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1709619709,
|
||||
"narHash": "sha256-l6EPVJfwfelWST7qWQeP6t/TDK3HHv5uUB1b2vw4mOQ=",
|
||||
"lastModified": 1745735608,
|
||||
"narHash": "sha256-L0jzm815XBFfF2wCFmR+M1CF+beIEFj6SxlqVKF59Ec=",
|
||||
"owner": "nix-community",
|
||||
"repo": "fenix",
|
||||
"rev": "c8943ea9e98d41325ff57d4ec14736d330b321b2",
|
||||
"rev": "c39a78eba6ed2a022cc3218db90d485077101496",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -87,22 +83,6 @@
|
|||
}
|
||||
},
|
||||
"flake-compat": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1673956053,
|
||||
"narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
|
||||
"owner": "edolstra",
|
||||
"repo": "flake-compat",
|
||||
"rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "edolstra",
|
||||
"repo": "flake-compat",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-compat_2": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1696426674,
|
||||
|
|
@ -118,31 +98,53 @@
|
|||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-utils": {
|
||||
"flake-compat_2": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1667395993,
|
||||
"narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
|
||||
"lastModified": 1733328505,
|
||||
"narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=",
|
||||
"owner": "edolstra",
|
||||
"repo": "flake-compat",
|
||||
"rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"owner": "edolstra",
|
||||
"repo": "flake-compat",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-utils_2": {
|
||||
"flake-parts": {
|
||||
"inputs": {
|
||||
"nixpkgs-lib": [
|
||||
"attic",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1722555600,
|
||||
"narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
|
||||
"owner": "hercules-ci",
|
||||
"repo": "flake-parts",
|
||||
"rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "hercules-ci",
|
||||
"repo": "flake-parts",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-utils": {
|
||||
"inputs": {
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1709126324,
|
||||
"narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=",
|
||||
"lastModified": 1731533236,
|
||||
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "d465f4819400de7c8d874d50b982301f28a84605",
|
||||
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -153,11 +155,11 @@
|
|||
},
|
||||
"nix-filter": {
|
||||
"locked": {
|
||||
"lastModified": 1705332318,
|
||||
"narHash": "sha256-kcw1yFeJe9N4PjQji9ZeX47jg0p9A0DuU4djKvg1a7I=",
|
||||
"lastModified": 1731533336,
|
||||
"narHash": "sha256-oRam5PS1vcrr5UPgALW0eo1m/5/pls27Z/pabHNy2Ms=",
|
||||
"owner": "numtide",
|
||||
"repo": "nix-filter",
|
||||
"rev": "3449dc925982ad46246cfc36469baf66e1b64f17",
|
||||
"rev": "f7653272fd234696ae94229839a99b73c9ab7de0",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -166,13 +168,34 @@
|
|||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nix-github-actions": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"attic",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1729742964,
|
||||
"narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
|
||||
"owner": "nix-community",
|
||||
"repo": "nix-github-actions",
|
||||
"rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-community",
|
||||
"repo": "nix-github-actions",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1702539185,
|
||||
"narHash": "sha256-KnIRG5NMdLIpEkZTnN5zovNYc0hhXjAgv6pfd5Z4c7U=",
|
||||
"lastModified": 1726042813,
|
||||
"narHash": "sha256-LnNKCCxnwgF+575y0pxUdlGZBO/ru1CtGHIqQVfvjlA=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "aa9d4729cbc99dabacb50e3994dcefb3ea0f7447",
|
||||
"rev": "159be5db480d1df880a0135ca0bfed84c2f88353",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -184,27 +207,27 @@
|
|||
},
|
||||
"nixpkgs-stable": {
|
||||
"locked": {
|
||||
"lastModified": 1702780907,
|
||||
"narHash": "sha256-blbrBBXjjZt6OKTcYX1jpe9SRof2P9ZYWPzq22tzXAA=",
|
||||
"lastModified": 1724316499,
|
||||
"narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "1e2e384c5b7c50dbf8e9c441a9e58d85f408b01f",
|
||||
"rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-23.11",
|
||||
"ref": "nixos-24.05",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs_2": {
|
||||
"locked": {
|
||||
"lastModified": 1709479366,
|
||||
"narHash": "sha256-n6F0n8UV6lnTZbYPl1A9q1BS0p4hduAv1mGAP17CVd0=",
|
||||
"lastModified": 1745526057,
|
||||
"narHash": "sha256-ITSpPDwvLBZBnPRS2bUcHY3gZSwis/uTe255QgMtTLA=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "b8697e57f10292a6165a20f03d2f42920dfaf973",
|
||||
"rev": "f771eb401a46846c1aebd20552521b233dd7e18b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -220,7 +243,7 @@
|
|||
"crane": "crane_2",
|
||||
"fenix": "fenix",
|
||||
"flake-compat": "flake-compat_2",
|
||||
"flake-utils": "flake-utils_2",
|
||||
"flake-utils": "flake-utils",
|
||||
"nix-filter": "nix-filter",
|
||||
"nixpkgs": "nixpkgs_2"
|
||||
}
|
||||
|
|
@ -228,11 +251,11 @@
|
|||
"rust-analyzer-src": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1709571018,
|
||||
"narHash": "sha256-ISFrxHxE0J5g7lDAscbK88hwaT5uewvWoma9TlFmRzM=",
|
||||
"lastModified": 1745694049,
|
||||
"narHash": "sha256-fxvRYH/tS7hGQeg9zCVh5RBcSWT+JGJet7RA8Ss+rC0=",
|
||||
"owner": "rust-lang",
|
||||
"repo": "rust-analyzer",
|
||||
"rev": "9f14343f9ee24f53f17492c5f9b653427e2ad15e",
|
||||
"rev": "d8887c0758bbd2d5f752d5bd405d4491e90e7ed6",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
|
|||
21
flake.nix
21
flake.nix
|
|
@ -12,10 +12,10 @@
|
|||
url = "github:nix-community/fenix";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
crane = {
|
||||
url = "github:ipetkov/crane?ref=master";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
# Pinned because crane's own automatic cross compilation configuration that they
|
||||
# introduce in the next commit attempts to link the musl targets against glibc
|
||||
# for some reason. Unpin once this is fixed.
|
||||
crane.url = "github:ipetkov/crane?rev=bb1c9567c43e4434f54e9481eb4b8e8e0d50f0b5";
|
||||
attic.url = "github:zhaofengli/attic?ref=main";
|
||||
};
|
||||
|
||||
|
|
@ -24,7 +24,7 @@
|
|||
# Keep sorted
|
||||
mkScope = pkgs: pkgs.lib.makeScope pkgs.newScope (self: {
|
||||
craneLib =
|
||||
(inputs.crane.mkLib pkgs).overrideToolchain self.toolchain;
|
||||
(inputs.crane.mkLib pkgs).overrideToolchain (_: self.toolchain);
|
||||
|
||||
default = self.callPackage ./nix/pkgs/default {};
|
||||
|
||||
|
|
@ -59,13 +59,20 @@
|
|||
file = ./rust-toolchain.toml;
|
||||
|
||||
# See also `rust-toolchain.toml`
|
||||
sha256 = "sha256-VZZnlyP69+Y3crrLHQyJirqlHrTtGTsyiSnZB8jEvVo=";
|
||||
sha256 = "sha256-s1RPtyvDGJaX/BisLT+ifVfuhDT1nZkZ1NcK8sbwELM=";
|
||||
};
|
||||
});
|
||||
in
|
||||
inputs.flake-utils.lib.eachDefaultSystem (system:
|
||||
let
|
||||
pkgs = inputs.nixpkgs.legacyPackages.${system};
|
||||
pkgs = (import inputs.nixpkgs {
|
||||
inherit system;
|
||||
|
||||
# libolm is deprecated, but we only need it for complement
|
||||
config.permittedInsecurePackages = [
|
||||
"olm-3.2.16"
|
||||
];
|
||||
});
|
||||
in
|
||||
{
|
||||
packages = {
|
||||
|
|
|
|||
|
|
@ -22,23 +22,10 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic {
|
|||
[ "-C" "relocation-model=static" ]
|
||||
++ lib.optionals
|
||||
(stdenv.buildPlatform.config != stdenv.hostPlatform.config)
|
||||
[ "-l" "c" ]
|
||||
++ lib.optionals
|
||||
# This check has to match the one [here][0]. We only need to set
|
||||
# these flags when using a different linker. Don't ask me why, though,
|
||||
# because I don't know. All I know is it breaks otherwise.
|
||||
#
|
||||
# [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L37-L40
|
||||
(
|
||||
# Nixpkgs doesn't check for x86_64 here but we do, because I
|
||||
# observed a failure building statically for x86_64 without
|
||||
# including it here. Linkers are weird.
|
||||
(stdenv.hostPlatform.isAarch64 || stdenv.hostPlatform.isx86_64)
|
||||
&& stdenv.hostPlatform.isStatic
|
||||
&& !stdenv.isDarwin
|
||||
&& !stdenv.cc.bintools.isLLVM
|
||||
)
|
||||
[
|
||||
"-l"
|
||||
"c"
|
||||
|
||||
"-l"
|
||||
"stdc++"
|
||||
"-L"
|
||||
|
|
@ -80,7 +67,7 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic {
|
|||
{
|
||||
"CC_${cargoEnvVarTarget}" = envVars.ccForHost;
|
||||
"CXX_${cargoEnvVarTarget}" = envVars.cxxForHost;
|
||||
"CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForHost;
|
||||
"CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.ccForHost;
|
||||
CARGO_BUILD_TARGET = rustcTarget;
|
||||
}
|
||||
)
|
||||
|
|
@ -92,7 +79,7 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic {
|
|||
{
|
||||
"CC_${cargoEnvVarTarget}" = envVars.ccForBuild;
|
||||
"CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild;
|
||||
"CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild;
|
||||
"CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.ccForBuild;
|
||||
HOST_CC = "${pkgsBuildHost.stdenv.cc}/bin/cc";
|
||||
HOST_CXX = "${pkgsBuildHost.stdenv.cc}/bin/c++";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ let
|
|||
let
|
||||
rocksdb' = rocksdb.override {
|
||||
enableJemalloc = builtins.elem "jemalloc" features;
|
||||
enableLiburing = false;
|
||||
};
|
||||
in
|
||||
{
|
||||
|
|
|
|||
|
|
@ -2,9 +2,18 @@
|
|||
{ default
|
||||
, dockerTools
|
||||
, lib
|
||||
, tini
|
||||
, pkgs
|
||||
}:
|
||||
|
||||
let
|
||||
# See https://github.com/krallin/tini/pull/223
|
||||
tini = pkgs.tini.overrideAttrs {
|
||||
patches = [ (pkgs.fetchpatch {
|
||||
url = "https://patch-diff.githubusercontent.com/raw/krallin/tini/pull/223.patch";
|
||||
hash = "sha256-i6xcf+qpjD+7ZQY3ueiDaxO4+UA2LutLCZLNmT+ji1s=";
|
||||
})
|
||||
];
|
||||
};
|
||||
in
|
||||
dockerTools.buildImage {
|
||||
name = default.pname;
|
||||
tag = "next";
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
# If you're having trouble making the relevant changes, bug a maintainer.
|
||||
|
||||
[toolchain]
|
||||
channel = "1.81.0"
|
||||
channel = "1.83.0"
|
||||
components = [
|
||||
# For rust-analyzer
|
||||
"rust-src",
|
||||
|
|
|
|||
|
|
@ -54,33 +54,34 @@ pub async fn get_media_config_auth_route(
|
|||
pub async fn create_content_route(
|
||||
body: Ruma<create_content::v3::Request>,
|
||||
) -> Result<create_content::v3::Response> {
|
||||
let mxc = format!(
|
||||
"mxc://{}/{}",
|
||||
services().globals.server_name(),
|
||||
utils::random_string(MXC_LENGTH)
|
||||
);
|
||||
let create_content::v3::Request {
|
||||
filename,
|
||||
content_type,
|
||||
file,
|
||||
..
|
||||
} = body.body;
|
||||
|
||||
let media_id = utils::random_string(MXC_LENGTH);
|
||||
|
||||
services()
|
||||
.media
|
||||
.create(
|
||||
mxc.clone(),
|
||||
Some(
|
||||
ContentDisposition::new(ContentDispositionType::Inline)
|
||||
.with_filename(body.filename.clone()),
|
||||
),
|
||||
body.content_type.as_deref(),
|
||||
&body.file,
|
||||
services().globals.server_name(),
|
||||
&media_id,
|
||||
filename.as_deref(),
|
||||
content_type.as_deref(),
|
||||
&file,
|
||||
body.sender_user.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(create_content::v3::Response {
|
||||
content_uri: mxc.into(),
|
||||
content_uri: (format!("mxc://{}/{}", services().globals.server_name(), media_id)).into(),
|
||||
blurhash: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn get_remote_content(
|
||||
mxc: &str,
|
||||
server_name: &ServerName,
|
||||
media_id: String,
|
||||
) -> Result<get_content::v1::Response, Error> {
|
||||
|
|
@ -120,7 +121,7 @@ pub async fn get_remote_content(
|
|||
server_name,
|
||||
media::get_content::v3::Request {
|
||||
server_name: server_name.to_owned(),
|
||||
media_id,
|
||||
media_id: media_id.clone(),
|
||||
timeout_ms: Duration::from_secs(20),
|
||||
allow_remote: false,
|
||||
allow_redirect: true,
|
||||
|
|
@ -140,10 +141,15 @@ pub async fn get_remote_content(
|
|||
services()
|
||||
.media
|
||||
.create(
|
||||
mxc.to_owned(),
|
||||
content_response.content_disposition.clone(),
|
||||
server_name,
|
||||
&media_id,
|
||||
content_response
|
||||
.content_disposition
|
||||
.as_ref()
|
||||
.and_then(|cd| cd.filename.as_deref()),
|
||||
content_response.content_type.as_deref(),
|
||||
&content_response.file,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
|
@ -162,7 +168,13 @@ pub async fn get_content_route(
|
|||
file,
|
||||
content_disposition,
|
||||
content_type,
|
||||
} = get_content(&body.server_name, body.media_id.clone(), body.allow_remote).await?;
|
||||
} = get_content(
|
||||
&body.server_name,
|
||||
body.media_id.clone(),
|
||||
body.allow_remote,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(media::get_content::v3::Response {
|
||||
file,
|
||||
|
|
@ -178,21 +190,25 @@ pub async fn get_content_route(
|
|||
pub async fn get_content_auth_route(
|
||||
body: Ruma<get_content::v1::Request>,
|
||||
) -> Result<get_content::v1::Response> {
|
||||
get_content(&body.server_name, body.media_id.clone(), true).await
|
||||
get_content(&body.server_name, body.media_id.clone(), true, true).await
|
||||
}
|
||||
|
||||
async fn get_content(
|
||||
pub async fn get_content(
|
||||
server_name: &ServerName,
|
||||
media_id: String,
|
||||
allow_remote: bool,
|
||||
authenticated: bool,
|
||||
) -> Result<get_content::v1::Response, Error> {
|
||||
let mxc = format!("mxc://{}/{}", server_name, media_id);
|
||||
services().media.check_blocked(server_name, &media_id)?;
|
||||
|
||||
if let Ok(Some(FileMeta {
|
||||
content_disposition,
|
||||
content_type,
|
||||
file,
|
||||
})) = services().media.get(mxc.clone()).await
|
||||
})) = services()
|
||||
.media
|
||||
.get(server_name, &media_id, authenticated)
|
||||
.await
|
||||
{
|
||||
Ok(get_content::v1::Response {
|
||||
file,
|
||||
|
|
@ -200,8 +216,7 @@ async fn get_content(
|
|||
content_disposition: Some(content_disposition),
|
||||
})
|
||||
} else if server_name != services().globals.server_name() && allow_remote {
|
||||
let remote_content_response =
|
||||
get_remote_content(&mxc, server_name, media_id.clone()).await?;
|
||||
let remote_content_response = get_remote_content(server_name, media_id.clone()).await?;
|
||||
|
||||
Ok(get_content::v1::Response {
|
||||
content_disposition: remote_content_response.content_disposition,
|
||||
|
|
@ -230,6 +245,7 @@ pub async fn get_content_as_filename_route(
|
|||
body.media_id.clone(),
|
||||
body.filename.clone(),
|
||||
body.allow_remote,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
|
@ -252,6 +268,7 @@ pub async fn get_content_as_filename_auth_route(
|
|||
body.media_id.clone(),
|
||||
body.filename.clone(),
|
||||
true,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
|
@ -261,12 +278,16 @@ async fn get_content_as_filename(
|
|||
media_id: String,
|
||||
filename: String,
|
||||
allow_remote: bool,
|
||||
authenticated: bool,
|
||||
) -> Result<get_content_as_filename::v1::Response, Error> {
|
||||
let mxc = format!("mxc://{}/{}", server_name, media_id);
|
||||
services().media.check_blocked(server_name, &media_id)?;
|
||||
|
||||
if let Ok(Some(FileMeta {
|
||||
file, content_type, ..
|
||||
})) = services().media.get(mxc.clone()).await
|
||||
})) = services()
|
||||
.media
|
||||
.get(server_name, &media_id, authenticated)
|
||||
.await
|
||||
{
|
||||
Ok(get_content_as_filename::v1::Response {
|
||||
file,
|
||||
|
|
@ -277,8 +298,7 @@ async fn get_content_as_filename(
|
|||
),
|
||||
})
|
||||
} else if server_name != services().globals.server_name() && allow_remote {
|
||||
let remote_content_response =
|
||||
get_remote_content(&mxc, server_name, media_id.clone()).await?;
|
||||
let remote_content_response = get_remote_content(server_name, media_id.clone()).await?;
|
||||
|
||||
Ok(get_content_as_filename::v1::Response {
|
||||
content_disposition: Some(
|
||||
|
|
@ -313,6 +333,7 @@ pub async fn get_content_thumbnail_route(
|
|||
body.method.clone(),
|
||||
body.animated,
|
||||
body.allow_remote,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
|
@ -338,10 +359,12 @@ pub async fn get_content_thumbnail_auth_route(
|
|||
body.method.clone(),
|
||||
body.animated,
|
||||
true,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn get_content_thumbnail(
|
||||
server_name: &ServerName,
|
||||
media_id: String,
|
||||
|
|
@ -350,8 +373,9 @@ async fn get_content_thumbnail(
|
|||
method: Option<Method>,
|
||||
animated: Option<bool>,
|
||||
allow_remote: bool,
|
||||
authenticated: bool,
|
||||
) -> Result<get_content_thumbnail::v1::Response, Error> {
|
||||
let mxc = format!("mxc://{}/{}", server_name, media_id);
|
||||
services().media.check_blocked(server_name, &media_id)?;
|
||||
|
||||
if let Some(FileMeta {
|
||||
file,
|
||||
|
|
@ -360,13 +384,15 @@ async fn get_content_thumbnail(
|
|||
}) = services()
|
||||
.media
|
||||
.get_thumbnail(
|
||||
mxc.clone(),
|
||||
server_name,
|
||||
&media_id,
|
||||
width
|
||||
.try_into()
|
||||
.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
|
||||
height
|
||||
.try_into()
|
||||
.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Height is invalid."))?,
|
||||
authenticated,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
|
|
@ -375,7 +401,7 @@ async fn get_content_thumbnail(
|
|||
content_type,
|
||||
content_disposition: Some(content_disposition),
|
||||
})
|
||||
} else if server_name != services().globals.server_name() && allow_remote {
|
||||
} else if server_name != services().globals.server_name() && allow_remote && authenticated {
|
||||
let thumbnail_response = match services()
|
||||
.sending
|
||||
.send_federation_request(
|
||||
|
|
@ -452,7 +478,12 @@ async fn get_content_thumbnail(
|
|||
services()
|
||||
.media
|
||||
.upload_thumbnail(
|
||||
mxc,
|
||||
server_name,
|
||||
&media_id,
|
||||
thumbnail_response
|
||||
.content_disposition
|
||||
.as_ref()
|
||||
.and_then(|cd| cd.filename.as_deref()),
|
||||
thumbnail_response.content_type.as_deref(),
|
||||
width.try_into().expect("all UInts are valid u32s"),
|
||||
height.try_into().expect("all UInts are valid u32s"),
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ mod device;
|
|||
mod directory;
|
||||
mod filter;
|
||||
mod keys;
|
||||
mod media;
|
||||
pub mod media;
|
||||
mod membership;
|
||||
mod message;
|
||||
mod openid;
|
||||
|
|
|
|||
|
|
@ -2215,23 +2215,24 @@ pub async fn create_invite_route(
|
|||
})
|
||||
}
|
||||
|
||||
/// # `GET /_matrix/federation/v1/media/download/{serverName}/{mediaId}`
|
||||
/// # `GET /_matrix/federation/v1/media/download/{mediaId}`
|
||||
///
|
||||
/// Load media from our server.
|
||||
pub async fn get_content_route(
|
||||
body: Ruma<get_content::v1::Request>,
|
||||
) -> Result<get_content::v1::Response> {
|
||||
let mxc = format!(
|
||||
"mxc://{}/{}",
|
||||
services().globals.server_name(),
|
||||
body.media_id
|
||||
);
|
||||
services()
|
||||
.media
|
||||
.check_blocked(services().globals.server_name(), &body.media_id)?;
|
||||
|
||||
if let Some(FileMeta {
|
||||
content_disposition,
|
||||
content_type,
|
||||
file,
|
||||
}) = services().media.get(mxc.clone()).await?
|
||||
}) = services()
|
||||
.media
|
||||
.get(services().globals.server_name(), &body.media_id, true)
|
||||
.await?
|
||||
{
|
||||
Ok(get_content::v1::Response::new(
|
||||
ContentMetadata::new(),
|
||||
|
|
@ -2246,17 +2247,15 @@ pub async fn get_content_route(
|
|||
}
|
||||
}
|
||||
|
||||
/// # `GET /_matrix/federation/v1/media/thumbnail/{serverName}/{mediaId}`
|
||||
/// # `GET /_matrix/federation/v1/media/thumbnail/{mediaId}`
|
||||
///
|
||||
/// Load media thumbnail from our server or over federation.
|
||||
pub async fn get_content_thumbnail_route(
|
||||
body: Ruma<get_content_thumbnail::v1::Request>,
|
||||
) -> Result<get_content_thumbnail::v1::Response> {
|
||||
let mxc = format!(
|
||||
"mxc://{}/{}",
|
||||
services().globals.server_name(),
|
||||
body.media_id
|
||||
);
|
||||
services()
|
||||
.media
|
||||
.check_blocked(services().globals.server_name(), &body.media_id)?;
|
||||
|
||||
let Some(FileMeta {
|
||||
file,
|
||||
|
|
@ -2265,13 +2264,15 @@ pub async fn get_content_thumbnail_route(
|
|||
}) = services()
|
||||
.media
|
||||
.get_thumbnail(
|
||||
mxc.clone(),
|
||||
services().globals.server_name(),
|
||||
&body.media_id,
|
||||
body.width
|
||||
.try_into()
|
||||
.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
|
||||
body.height
|
||||
.try_into()
|
||||
.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
|
||||
true,
|
||||
)
|
||||
.await?
|
||||
else {
|
||||
|
|
@ -2281,7 +2282,9 @@ pub async fn get_content_thumbnail_route(
|
|||
services()
|
||||
.media
|
||||
.upload_thumbnail(
|
||||
mxc,
|
||||
services().globals.server_name(),
|
||||
&body.media_id,
|
||||
content_disposition.filename.as_deref(),
|
||||
content_type.as_deref(),
|
||||
body.width.try_into().expect("all UInts are valid u32s"),
|
||||
body.height.try_into().expect("all UInts are valid u32s"),
|
||||
|
|
|
|||
|
|
@ -1,20 +1,28 @@
|
|||
use std::{
|
||||
collections::BTreeMap,
|
||||
collections::{BTreeMap, HashMap, HashSet},
|
||||
fmt,
|
||||
net::{IpAddr, Ipv4Addr},
|
||||
num::NonZeroU8,
|
||||
path::PathBuf,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use bytesize::ByteSize;
|
||||
use ruma::{OwnedServerName, RoomVersionId};
|
||||
use serde::{de::IgnoredAny, Deserialize};
|
||||
use tokio::time::{interval, Interval};
|
||||
use tracing::warn;
|
||||
use url::Url;
|
||||
|
||||
mod proxy;
|
||||
use crate::Error;
|
||||
|
||||
mod proxy;
|
||||
use self::proxy::ProxyConfig;
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
pub struct Config {
|
||||
const SHA256_HEX_LENGTH: u8 = 64;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct IncompleteConfig {
|
||||
#[serde(default = "default_address")]
|
||||
pub address: IpAddr,
|
||||
#[serde(default = "default_port")]
|
||||
|
|
@ -60,7 +68,7 @@ pub struct Config {
|
|||
#[serde(default = "default_default_room_version")]
|
||||
pub default_room_version: RoomVersionId,
|
||||
#[serde(default)]
|
||||
pub well_known: WellKnownConfig,
|
||||
pub well_known: IncompleteWellKnownConfig,
|
||||
#[serde(default = "false_fn")]
|
||||
pub allow_jaeger: bool,
|
||||
#[serde(default = "false_fn")]
|
||||
|
|
@ -81,12 +89,203 @@ pub struct Config {
|
|||
|
||||
pub turn: Option<TurnConfig>,
|
||||
|
||||
#[serde(default)]
|
||||
pub media: IncompleteMediaConfig,
|
||||
|
||||
pub emergency_password: Option<String>,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub catchall: BTreeMap<String, IgnoredAny>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Clone, Debug)]
|
||||
#[serde(from = "IncompleteConfig")]
|
||||
pub struct Config {
|
||||
pub address: IpAddr,
|
||||
pub port: u16,
|
||||
pub tls: Option<TlsConfig>,
|
||||
|
||||
pub server_name: OwnedServerName,
|
||||
pub database_backend: String,
|
||||
pub database_path: String,
|
||||
pub db_cache_capacity_mb: f64,
|
||||
pub enable_lightning_bolt: bool,
|
||||
pub allow_check_for_updates: bool,
|
||||
pub conduit_cache_capacity_modifier: f64,
|
||||
pub rocksdb_max_open_files: i32,
|
||||
pub pdu_cache_capacity: u32,
|
||||
pub cleanup_second_interval: u32,
|
||||
pub max_request_size: u32,
|
||||
pub max_concurrent_requests: u16,
|
||||
pub max_fetch_prev_events: u16,
|
||||
pub allow_registration: bool,
|
||||
pub registration_token: Option<String>,
|
||||
pub openid_token_ttl: u64,
|
||||
pub allow_encryption: bool,
|
||||
pub allow_federation: bool,
|
||||
pub allow_room_creation: bool,
|
||||
pub allow_unstable_room_versions: bool,
|
||||
pub default_room_version: RoomVersionId,
|
||||
pub well_known: WellKnownConfig,
|
||||
pub allow_jaeger: bool,
|
||||
pub tracing_flame: bool,
|
||||
pub proxy: ProxyConfig,
|
||||
pub jwt_secret: Option<String>,
|
||||
pub trusted_servers: Vec<OwnedServerName>,
|
||||
pub log: String,
|
||||
|
||||
pub turn: Option<TurnConfig>,
|
||||
|
||||
pub media: MediaConfig,
|
||||
|
||||
pub emergency_password: Option<String>,
|
||||
|
||||
pub catchall: BTreeMap<String, IgnoredAny>,
|
||||
}
|
||||
|
||||
impl From<IncompleteConfig> for Config {
|
||||
fn from(val: IncompleteConfig) -> Self {
|
||||
let IncompleteConfig {
|
||||
address,
|
||||
port,
|
||||
tls,
|
||||
server_name,
|
||||
database_backend,
|
||||
database_path,
|
||||
db_cache_capacity_mb,
|
||||
enable_lightning_bolt,
|
||||
allow_check_for_updates,
|
||||
conduit_cache_capacity_modifier,
|
||||
rocksdb_max_open_files,
|
||||
pdu_cache_capacity,
|
||||
cleanup_second_interval,
|
||||
max_request_size,
|
||||
max_concurrent_requests,
|
||||
max_fetch_prev_events,
|
||||
allow_registration,
|
||||
registration_token,
|
||||
openid_token_ttl,
|
||||
allow_encryption,
|
||||
allow_federation,
|
||||
allow_room_creation,
|
||||
allow_unstable_room_versions,
|
||||
default_room_version,
|
||||
well_known,
|
||||
allow_jaeger,
|
||||
tracing_flame,
|
||||
proxy,
|
||||
jwt_secret,
|
||||
trusted_servers,
|
||||
log,
|
||||
turn_username,
|
||||
turn_password,
|
||||
turn_uris,
|
||||
turn_secret,
|
||||
turn_ttl,
|
||||
turn,
|
||||
media,
|
||||
emergency_password,
|
||||
catchall,
|
||||
} = val;
|
||||
|
||||
let turn = turn.or_else(|| {
|
||||
let auth = if let Some(secret) = turn_secret {
|
||||
TurnAuth::Secret { secret }
|
||||
} else if let (Some(username), Some(password)) = (turn_username, turn_password) {
|
||||
TurnAuth::UserPass { username, password }
|
||||
} else {
|
||||
return None;
|
||||
};
|
||||
|
||||
if let (Some(uris), ttl) = (turn_uris, turn_ttl) {
|
||||
Some(TurnConfig { uris, ttl, auth })
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
|
||||
let well_known_client = well_known
|
||||
.client
|
||||
.map(String::from)
|
||||
.unwrap_or_else(|| format!("https://{server_name}"));
|
||||
|
||||
let well_known_server = well_known.server.unwrap_or_else(|| {
|
||||
if server_name.port().is_some() {
|
||||
server_name.clone()
|
||||
} else {
|
||||
format!("{}:443", server_name.host())
|
||||
.try_into()
|
||||
.expect("Host from valid hostname + :443 must be valid")
|
||||
}
|
||||
});
|
||||
|
||||
let well_known = WellKnownConfig {
|
||||
client: well_known_client,
|
||||
server: well_known_server,
|
||||
};
|
||||
|
||||
let media = MediaConfig {
|
||||
backend: match media.backend {
|
||||
IncompleteMediaBackendConfig::FileSystem {
|
||||
path,
|
||||
directory_structure,
|
||||
} => MediaBackendConfig::FileSystem {
|
||||
path: path.unwrap_or_else(|| {
|
||||
// We do this as we don't know if the path has a trailing slash, or even if the
|
||||
// path separator is a forward or backward slash
|
||||
[&database_path, "media"]
|
||||
.iter()
|
||||
.collect::<PathBuf>()
|
||||
.into_os_string()
|
||||
.into_string()
|
||||
.expect("Both inputs are valid UTF-8")
|
||||
}),
|
||||
directory_structure,
|
||||
},
|
||||
},
|
||||
retention: media.retention.into(),
|
||||
};
|
||||
|
||||
Config {
|
||||
address,
|
||||
port,
|
||||
tls,
|
||||
server_name,
|
||||
database_backend,
|
||||
database_path,
|
||||
db_cache_capacity_mb,
|
||||
enable_lightning_bolt,
|
||||
allow_check_for_updates,
|
||||
conduit_cache_capacity_modifier,
|
||||
rocksdb_max_open_files,
|
||||
pdu_cache_capacity,
|
||||
cleanup_second_interval,
|
||||
max_request_size,
|
||||
max_concurrent_requests,
|
||||
max_fetch_prev_events,
|
||||
allow_registration,
|
||||
registration_token,
|
||||
openid_token_ttl,
|
||||
allow_encryption,
|
||||
allow_federation,
|
||||
allow_room_creation,
|
||||
allow_unstable_room_versions,
|
||||
default_room_version,
|
||||
well_known,
|
||||
allow_jaeger,
|
||||
tracing_flame,
|
||||
proxy,
|
||||
jwt_secret,
|
||||
trusted_servers,
|
||||
log,
|
||||
turn,
|
||||
media,
|
||||
emergency_password,
|
||||
catchall,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
pub struct TlsConfig {
|
||||
pub certs: String,
|
||||
|
|
@ -110,11 +309,251 @@ pub enum TurnAuth {
|
|||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Default)]
|
||||
pub struct WellKnownConfig {
|
||||
pub struct IncompleteWellKnownConfig {
|
||||
// We use URL here so that the user gets an error if the config isn't a valid url
|
||||
pub client: Option<Url>,
|
||||
pub server: Option<OwnedServerName>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct WellKnownConfig {
|
||||
// We use String here as there is no point converting our manually constructed String into a
|
||||
// URL, just for it to be converted back into a &str
|
||||
pub client: String,
|
||||
pub server: OwnedServerName,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Default)]
|
||||
pub struct IncompleteMediaConfig {
|
||||
#[serde(flatten, default)]
|
||||
pub backend: IncompleteMediaBackendConfig,
|
||||
pub retention: IncompleteMediaRetentionConfig,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct MediaConfig {
|
||||
pub backend: MediaBackendConfig,
|
||||
pub retention: MediaRetentionConfig,
|
||||
}
|
||||
|
||||
type IncompleteMediaRetentionConfig = Option<HashSet<IncompleteScopedMediaRetentionConfig>>;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct MediaRetentionConfig {
|
||||
pub scoped: HashMap<MediaRetentionScope, ScopedMediaRetentionConfig>,
|
||||
pub global_space: Option<ByteSize>,
|
||||
}
|
||||
|
||||
impl MediaRetentionConfig {
|
||||
/// Interval for the duration-based retention policies to be checked & enforced
|
||||
pub fn cleanup_interval(&self) -> Option<Interval> {
|
||||
self.scoped
|
||||
.values()
|
||||
.filter_map(|scoped| match (scoped.created, scoped.accessed) {
|
||||
(None, accessed) => accessed,
|
||||
(created, None) => created,
|
||||
(created, accessed) => created.min(accessed),
|
||||
})
|
||||
.map(|dur| {
|
||||
dur.mul_f32(0.1)
|
||||
.max(Duration::from_secs(60).min(Duration::from_secs(60 * 60 * 24)))
|
||||
})
|
||||
.min()
|
||||
.map(interval)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct IncompleteScopedMediaRetentionConfig {
|
||||
pub scope: Option<MediaRetentionScope>,
|
||||
#[serde(default, with = "humantime_serde::option")]
|
||||
pub accessed: Option<Duration>,
|
||||
#[serde(default, with = "humantime_serde::option")]
|
||||
pub created: Option<Duration>,
|
||||
pub space: Option<ByteSize>,
|
||||
}
|
||||
|
||||
impl From<IncompleteMediaRetentionConfig> for MediaRetentionConfig {
|
||||
fn from(value: IncompleteMediaRetentionConfig) -> Self {
|
||||
{
|
||||
let mut scoped = HashMap::from([
|
||||
(
|
||||
MediaRetentionScope::Remote,
|
||||
ScopedMediaRetentionConfig::default(),
|
||||
),
|
||||
(
|
||||
MediaRetentionScope::Thumbnail,
|
||||
ScopedMediaRetentionConfig::default(),
|
||||
),
|
||||
]);
|
||||
let mut fallback = None;
|
||||
|
||||
if let Some(retention) = value {
|
||||
for IncompleteScopedMediaRetentionConfig {
|
||||
scope,
|
||||
accessed,
|
||||
space,
|
||||
created,
|
||||
} in retention
|
||||
{
|
||||
if let Some(scope) = scope {
|
||||
scoped.insert(
|
||||
scope,
|
||||
ScopedMediaRetentionConfig {
|
||||
accessed,
|
||||
space,
|
||||
created,
|
||||
},
|
||||
);
|
||||
} else {
|
||||
fallback = Some(ScopedMediaRetentionConfig {
|
||||
accessed,
|
||||
space,
|
||||
created,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(fallback) = fallback.clone() {
|
||||
for scope in [
|
||||
MediaRetentionScope::Remote,
|
||||
MediaRetentionScope::Local,
|
||||
MediaRetentionScope::Thumbnail,
|
||||
] {
|
||||
scoped.entry(scope).or_insert_with(|| fallback.clone());
|
||||
}
|
||||
}
|
||||
|
||||
Self {
|
||||
global_space: fallback.and_then(|global| global.space),
|
||||
scoped,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::hash::Hash for IncompleteScopedMediaRetentionConfig {
|
||||
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
|
||||
self.scope.hash(state);
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for IncompleteScopedMediaRetentionConfig {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.scope == other.scope
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for IncompleteScopedMediaRetentionConfig {}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScopedMediaRetentionConfig {
|
||||
pub accessed: Option<Duration>,
|
||||
pub created: Option<Duration>,
|
||||
pub space: Option<ByteSize>,
|
||||
}
|
||||
|
||||
impl Default for ScopedMediaRetentionConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
// 30 days
|
||||
accessed: Some(Duration::from_secs(60 * 60 * 24 * 30)),
|
||||
created: None,
|
||||
space: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Clone, Debug, Hash, PartialEq, Eq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum MediaRetentionScope {
|
||||
Remote,
|
||||
Local,
|
||||
Thumbnail,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(tag = "backend", rename_all = "lowercase")]
|
||||
pub enum IncompleteMediaBackendConfig {
|
||||
FileSystem {
|
||||
path: Option<String>,
|
||||
#[serde(default)]
|
||||
directory_structure: DirectoryStructure,
|
||||
},
|
||||
}
|
||||
|
||||
impl Default for IncompleteMediaBackendConfig {
|
||||
fn default() -> Self {
|
||||
Self::FileSystem {
|
||||
path: None,
|
||||
directory_structure: DirectoryStructure::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum MediaBackendConfig {
|
||||
FileSystem {
|
||||
path: String,
|
||||
directory_structure: DirectoryStructure,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
// See https://github.com/serde-rs/serde/issues/642#issuecomment-525432907
|
||||
#[serde(try_from = "ShadowDirectoryStructure", untagged)]
|
||||
pub enum DirectoryStructure {
|
||||
// We do this enum instead of Option<DirectoryStructure>, so that we can have the structure be
|
||||
// deep by default, while still providing a away for it to be flat (by creating an empty table)
|
||||
//
|
||||
// e.g.:
|
||||
// ```toml
|
||||
// [global.media.directory_structure]
|
||||
// ```
|
||||
Flat,
|
||||
Deep { length: NonZeroU8, depth: NonZeroU8 },
|
||||
}
|
||||
|
||||
impl Default for DirectoryStructure {
|
||||
fn default() -> Self {
|
||||
Self::Deep {
|
||||
length: NonZeroU8::new(2).expect("2 is not 0"),
|
||||
depth: NonZeroU8::new(2).expect("2 is not 0"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum ShadowDirectoryStructure {
|
||||
Flat {},
|
||||
Deep { length: NonZeroU8, depth: NonZeroU8 },
|
||||
}
|
||||
|
||||
impl TryFrom<ShadowDirectoryStructure> for DirectoryStructure {
|
||||
type Error = Error;
|
||||
|
||||
fn try_from(value: ShadowDirectoryStructure) -> Result<Self, Self::Error> {
|
||||
match value {
|
||||
ShadowDirectoryStructure::Flat {} => Ok(Self::Flat),
|
||||
ShadowDirectoryStructure::Deep { length, depth } => {
|
||||
if length
|
||||
.get()
|
||||
.checked_mul(depth.get())
|
||||
.map(|product| product < SHA256_HEX_LENGTH)
|
||||
// If an overflow occurs, it definitely isn't less than SHA256_HEX_LENGTH
|
||||
.unwrap_or(false)
|
||||
{
|
||||
Ok(Self::Deep { length, depth })
|
||||
} else {
|
||||
Err(Error::bad_config("The media directory structure depth multiplied by the depth is equal to or greater than a sha256 hex hash, please reduce at least one of the two so that their product is less than 64"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const DEPRECATED_KEYS: &[&str] = &[
|
||||
"cache_capacity",
|
||||
"turn_username",
|
||||
|
|
@ -142,61 +581,9 @@ impl Config {
|
|||
}
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn well_known_client(&self) -> String {
|
||||
if let Some(url) = &self.well_known.client {
|
||||
url.to_string()
|
||||
} else {
|
||||
format!("https://{}", self.server_name)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn well_known_server(&self) -> OwnedServerName {
|
||||
match &self.well_known.server {
|
||||
Some(server_name) => server_name.to_owned(),
|
||||
None => {
|
||||
if self.server_name.port().is_some() {
|
||||
self.server_name.to_owned()
|
||||
} else {
|
||||
format!("{}:443", self.server_name.host())
|
||||
.try_into()
|
||||
.expect("Host from valid hostname + :443 must be valid")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn turn(&self) -> Option<TurnConfig> {
|
||||
if self.turn.is_some() {
|
||||
self.turn.clone()
|
||||
} else if let Some(uris) = self.turn_uris.clone() {
|
||||
if let Some(secret) = self.turn_secret.clone() {
|
||||
Some(TurnConfig {
|
||||
uris,
|
||||
ttl: self.turn_ttl,
|
||||
auth: TurnAuth::Secret { secret },
|
||||
})
|
||||
} else if let (Some(username), Some(password)) =
|
||||
(self.turn_username.clone(), self.turn_password.clone())
|
||||
{
|
||||
Some(TurnConfig {
|
||||
uris,
|
||||
ttl: self.turn_ttl,
|
||||
auth: TurnAuth::UserPass { username, password },
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Config {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
// Prepare a list of config values to show
|
||||
let well_known_server = self.well_known_server();
|
||||
let lines = [
|
||||
("Server name", self.server_name.host()),
|
||||
("Database backend", &self.database_backend),
|
||||
|
|
@ -247,7 +634,7 @@ impl fmt::Display for Config {
|
|||
&lst.join(", ")
|
||||
}),
|
||||
("TURN URIs", {
|
||||
if let Some(turn) = self.turn() {
|
||||
if let Some(turn) = &self.turn {
|
||||
let mut lst = vec![];
|
||||
for item in turn.uris.iter().cloned().enumerate() {
|
||||
let (_, uri): (usize, String) = item;
|
||||
|
|
@ -258,8 +645,8 @@ impl fmt::Display for Config {
|
|||
"unset"
|
||||
}
|
||||
}),
|
||||
("Well-known server name", well_known_server.as_str()),
|
||||
("Well-known client URL", &self.well_known_client()),
|
||||
("Well-known server name", self.well_known.server.as_str()),
|
||||
("Well-known client URL", &self.well_known.client),
|
||||
];
|
||||
|
||||
let mut msg: String = "Active config values:\n\n".to_owned();
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -3,7 +3,7 @@ mod account_data;
|
|||
mod appservice;
|
||||
mod globals;
|
||||
mod key_backups;
|
||||
mod media;
|
||||
pub(super) mod media;
|
||||
//mod pdu;
|
||||
mod pusher;
|
||||
mod rooms;
|
||||
|
|
|
|||
|
|
@ -89,11 +89,11 @@ impl service::sending::Data for KeyValueDatabase {
|
|||
outgoing_kind: &OutgoingKind,
|
||||
) -> Box<dyn Iterator<Item = Result<(SendingEventType, Vec<u8>)>> + 'a> {
|
||||
let prefix = outgoing_kind.get_prefix();
|
||||
return Box::new(
|
||||
Box::new(
|
||||
self.servernameevent_data
|
||||
.scan_prefix(prefix)
|
||||
.map(|(k, v)| parse_servercurrentevent(&k, v).map(|(_, ev)| (ev, k))),
|
||||
);
|
||||
)
|
||||
}
|
||||
|
||||
fn mark_as_active(&self, events: &[(SendingEventType, Vec<u8>)]) -> Result<()> {
|
||||
|
|
|
|||
|
|
@ -2,12 +2,13 @@ pub mod abstraction;
|
|||
pub mod key_value;
|
||||
|
||||
use crate::{
|
||||
service::rooms::timeline::PduCount, services, utils, Config, Error, PduEvent, Result, Services,
|
||||
SERVICES,
|
||||
service::{globals, rooms::timeline::PduCount},
|
||||
services, utils, Config, Error, PduEvent, Result, Services, SERVICES,
|
||||
};
|
||||
use abstraction::{KeyValueDatabaseEngine, KvTree};
|
||||
use base64::{engine::general_purpose, Engine};
|
||||
use directories::ProjectDirs;
|
||||
use key_value::media::FilehashMetadata;
|
||||
use lru_cache::LruCache;
|
||||
|
||||
use ruma::{
|
||||
|
|
@ -17,23 +18,50 @@ use ruma::{
|
|||
GlobalAccountDataEvent, GlobalAccountDataEventType, StateEventType,
|
||||
},
|
||||
push::Ruleset,
|
||||
CanonicalJsonValue, EventId, OwnedDeviceId, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId,
|
||||
UserId,
|
||||
CanonicalJsonValue, EventId, OwnedDeviceId, OwnedEventId, OwnedMxcUri, OwnedRoomId,
|
||||
OwnedUserId, RoomId, UserId,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{
|
||||
collections::{BTreeMap, HashMap, HashSet},
|
||||
fs::{self, remove_dir_all},
|
||||
io::Write,
|
||||
mem::size_of,
|
||||
path::Path,
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Mutex, RwLock},
|
||||
time::Duration,
|
||||
time::{Duration, UNIX_EPOCH},
|
||||
};
|
||||
use tokio::time::interval;
|
||||
use tokio::{io::AsyncReadExt, time::interval};
|
||||
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
/// This trait should only be used for migrations, and hence should never be made "pub"
|
||||
trait GlobalsMigrationsExt {
|
||||
/// As the name states, old version of `get_media_file`, only for usage in migrations
|
||||
fn get_media_file_old_only_use_for_migrations(&self, key: &[u8]) -> PathBuf;
|
||||
|
||||
/// As the name states, this should only be used for migrations.
|
||||
fn get_media_folder_only_use_for_migrations(&self) -> PathBuf;
|
||||
}
|
||||
|
||||
impl GlobalsMigrationsExt for globals::Service {
|
||||
fn get_media_file_old_only_use_for_migrations(&self, key: &[u8]) -> PathBuf {
|
||||
let mut r = PathBuf::new();
|
||||
r.push(self.config.database_path.clone());
|
||||
r.push("media");
|
||||
r.push(general_purpose::URL_SAFE_NO_PAD.encode(key));
|
||||
r
|
||||
}
|
||||
|
||||
fn get_media_folder_only_use_for_migrations(&self) -> PathBuf {
|
||||
let mut r = PathBuf::new();
|
||||
r.push(self.config.database_path.clone());
|
||||
r.push("media");
|
||||
r
|
||||
}
|
||||
}
|
||||
|
||||
pub struct KeyValueDatabase {
|
||||
_db: Arc<dyn KeyValueDatabaseEngine>,
|
||||
|
||||
|
|
@ -148,7 +176,14 @@ pub struct KeyValueDatabase {
|
|||
pub(super) roomusertype_roomuserdataid: Arc<dyn KvTree>, // RoomUserType = Room + User + Type
|
||||
|
||||
//pub media: media::Media,
|
||||
pub(super) mediaid_file: Arc<dyn KvTree>, // MediaId = MXC + WidthHeight + ContentDisposition + ContentType
|
||||
pub(super) servernamemediaid_metadata: Arc<dyn KvTree>, // Servername + MediaID -> content sha256 + Filename + ContentType + extra 0xff byte if media is allowed on unauthenticated endpoints
|
||||
pub(super) filehash_servername_mediaid: Arc<dyn KvTree>, // sha256 of content + Servername + MediaID, used to delete dangling references to filehashes from servernamemediaid
|
||||
pub(super) filehash_metadata: Arc<dyn KvTree>, // sha256 of content -> file size + creation time + last access time
|
||||
pub(super) blocked_servername_mediaid: Arc<dyn KvTree>, // Servername + MediaID of blocked media -> time of block + reason
|
||||
pub(super) servername_userlocalpart_mediaid: Arc<dyn KvTree>, // Servername + User Localpart + MediaID
|
||||
pub(super) servernamemediaid_userlocalpart: Arc<dyn KvTree>, // Servername + MediaID -> User Localpart, used to remove keys from above when files are deleted by unrelated means
|
||||
pub(super) thumbnailid_metadata: Arc<dyn KvTree>, // ThumbnailId = Servername + MediaID + width + height -> Filename + ContentType + extra 0xff byte if media is allowed on unauthenticated endpoints
|
||||
pub(super) filehash_thumbnailid: Arc<dyn KvTree>, // sha256 of content + "ThumbnailId", as defined above. Used to dangling references to filehashes from thumbnailIds
|
||||
//pub key_backups: key_backups::KeyBackups,
|
||||
pub(super) backupid_algorithm: Arc<dyn KvTree>, // BackupId = UserId + Version(Count)
|
||||
pub(super) backupid_etag: Arc<dyn KvTree>, // BackupId = UserId + Version(Count)
|
||||
|
|
@ -352,7 +387,16 @@ impl KeyValueDatabase {
|
|||
referencedevents: builder.open_tree("referencedevents")?,
|
||||
roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?,
|
||||
roomusertype_roomuserdataid: builder.open_tree("roomusertype_roomuserdataid")?,
|
||||
mediaid_file: builder.open_tree("mediaid_file")?,
|
||||
servernamemediaid_metadata: builder.open_tree("servernamemediaid_metadata")?,
|
||||
filehash_servername_mediaid: builder.open_tree("filehash_servername_mediaid")?,
|
||||
filehash_metadata: builder.open_tree("filehash_metadata")?,
|
||||
blocked_servername_mediaid: builder.open_tree("blocked_servername_mediaid")?,
|
||||
servername_userlocalpart_mediaid: builder
|
||||
.open_tree("servername_userlocalpart_mediaid")?,
|
||||
servernamemediaid_userlocalpart: builder
|
||||
.open_tree("servernamemediaid_userlocalpart")?,
|
||||
thumbnailid_metadata: builder.open_tree("thumbnailid_metadata")?,
|
||||
filehash_thumbnailid: builder.open_tree("filehash_thumbnailid")?,
|
||||
backupid_algorithm: builder.open_tree("backupid_algorithm")?,
|
||||
backupid_etag: builder.open_tree("backupid_etag")?,
|
||||
backupkeyid_backup: builder.open_tree("backupkeyid_backup")?,
|
||||
|
|
@ -415,7 +459,7 @@ impl KeyValueDatabase {
|
|||
}
|
||||
|
||||
// If the database has any data, perform data migrations before starting
|
||||
let latest_database_version = 16;
|
||||
let latest_database_version = 17;
|
||||
|
||||
if services().users.count()? > 0 {
|
||||
// MIGRATIONS
|
||||
|
|
@ -462,16 +506,19 @@ impl KeyValueDatabase {
|
|||
}
|
||||
|
||||
if services().globals.database_version()? < 3 {
|
||||
let tree = db._db.open_tree("mediaid_file")?;
|
||||
// Move media to filesystem
|
||||
for (key, content) in db.mediaid_file.iter() {
|
||||
for (key, content) in tree.iter() {
|
||||
if content.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let path = services().globals.get_media_file(&key);
|
||||
let path = services()
|
||||
.globals
|
||||
.get_media_file_old_only_use_for_migrations(&key);
|
||||
let mut file = fs::File::create(path)?;
|
||||
file.write_all(&content)?;
|
||||
db.mediaid_file.insert(&key, &[])?;
|
||||
tree.insert(&key, &[])?;
|
||||
}
|
||||
|
||||
services().globals.bump_database_version(3)?;
|
||||
|
|
@ -933,16 +980,23 @@ impl KeyValueDatabase {
|
|||
}
|
||||
|
||||
if services().globals.database_version()? < 16 {
|
||||
let tree = db._db.open_tree("mediaid_file")?;
|
||||
// Reconstruct all media using the filesystem
|
||||
db.mediaid_file.clear().unwrap();
|
||||
tree.clear().unwrap();
|
||||
|
||||
for file in fs::read_dir(services().globals.get_media_folder()).unwrap() {
|
||||
for file in fs::read_dir(
|
||||
services()
|
||||
.globals
|
||||
.get_media_folder_only_use_for_migrations(),
|
||||
)
|
||||
.unwrap()
|
||||
{
|
||||
let file = file.unwrap();
|
||||
let file_name = file.file_name().into_string().unwrap();
|
||||
|
||||
let mediaid = general_purpose::URL_SAFE_NO_PAD.decode(&file_name).unwrap();
|
||||
|
||||
if let Err(e) = migrate_content_disposition_format(mediaid, db) {
|
||||
if let Err(e) = migrate_content_disposition_format(mediaid, &tree) {
|
||||
error!("Error migrating media file with name \"{file_name}\": {e}");
|
||||
return Err(e);
|
||||
}
|
||||
|
|
@ -952,6 +1006,55 @@ impl KeyValueDatabase {
|
|||
warn!("Migration: 13 -> 16 finished");
|
||||
}
|
||||
|
||||
if services().globals.database_version()? < 17 {
|
||||
warn!("Migrating media repository to new format. If you have a lot of media stored, this may take a while, so please be patiant!");
|
||||
|
||||
let tree = db._db.open_tree("mediaid_file")?;
|
||||
tree.clear().unwrap();
|
||||
|
||||
let mxc_prefix = general_purpose::URL_SAFE_NO_PAD.encode(b"mxc://");
|
||||
for file in fs::read_dir(
|
||||
services()
|
||||
.globals
|
||||
.get_media_folder_only_use_for_migrations(),
|
||||
)
|
||||
.unwrap()
|
||||
.filter_map(Result::ok)
|
||||
.filter(|result| {
|
||||
result.file_type().unwrap().is_file()
|
||||
&& result
|
||||
.file_name()
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.starts_with(&mxc_prefix)
|
||||
}) {
|
||||
let file_name = file.file_name().into_string().unwrap();
|
||||
|
||||
if let Err(e) = migrate_to_sha256_media(
|
||||
db,
|
||||
&file_name,
|
||||
file.metadata()
|
||||
.ok()
|
||||
.and_then(|meta| meta.created().ok())
|
||||
.and_then(|time| time.duration_since(UNIX_EPOCH).ok())
|
||||
.map(|dur| dur.as_secs()),
|
||||
file.metadata()
|
||||
.ok()
|
||||
.and_then(|meta| meta.accessed().ok())
|
||||
.and_then(|time| time.duration_since(UNIX_EPOCH).ok())
|
||||
.map(|dur| dur.as_secs()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!("Error migrating media file with name \"{file_name}\": {e}");
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
services().globals.bump_database_version(17)?;
|
||||
|
||||
warn!("Migration: 16 -> 17 finished");
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
services().globals.database_version().unwrap(),
|
||||
latest_database_version
|
||||
|
|
@ -1000,6 +1103,8 @@ impl KeyValueDatabase {
|
|||
|
||||
services().sending.start_handler();
|
||||
|
||||
services().media.start_time_retention_checker();
|
||||
|
||||
Self::start_cleanup_task().await;
|
||||
if services().globals.allow_check_for_updates() {
|
||||
Self::start_check_for_updates_task();
|
||||
|
|
@ -1117,7 +1222,7 @@ impl KeyValueDatabase {
|
|||
|
||||
fn migrate_content_disposition_format(
|
||||
mediaid: Vec<u8>,
|
||||
db: &KeyValueDatabase,
|
||||
tree: &Arc<dyn KvTree>,
|
||||
) -> Result<(), Error> {
|
||||
let mut parts = mediaid.rsplit(|&b| b == 0xff);
|
||||
let mut removed_bytes = 0;
|
||||
|
|
@ -1153,28 +1258,165 @@ fn migrate_content_disposition_format(
|
|||
|
||||
// Some file names are too long. Ignore those.
|
||||
match fs::rename(
|
||||
services().globals.get_media_file(&mediaid),
|
||||
services().globals.get_media_file(&new_key),
|
||||
services()
|
||||
.globals
|
||||
.get_media_file_old_only_use_for_migrations(&mediaid),
|
||||
services()
|
||||
.globals
|
||||
.get_media_file_old_only_use_for_migrations(&new_key),
|
||||
) {
|
||||
Ok(_) => {
|
||||
db.mediaid_file.insert(&new_key, &[])?;
|
||||
tree.insert(&new_key, &[])?;
|
||||
}
|
||||
Err(_) => {
|
||||
fs::rename(
|
||||
services().globals.get_media_file(&mediaid),
|
||||
services().globals.get_media_file(&shorter_key),
|
||||
services()
|
||||
.globals
|
||||
.get_media_file_old_only_use_for_migrations(&mediaid),
|
||||
services()
|
||||
.globals
|
||||
.get_media_file_old_only_use_for_migrations(&shorter_key),
|
||||
)
|
||||
.unwrap();
|
||||
db.mediaid_file.insert(&shorter_key, &[])?;
|
||||
tree.insert(&shorter_key, &[])?;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
db.mediaid_file.insert(&mediaid, &[])?;
|
||||
tree.insert(&mediaid, &[])?;
|
||||
};
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn migrate_to_sha256_media(
|
||||
db: &KeyValueDatabase,
|
||||
file_name: &str,
|
||||
creation: Option<u64>,
|
||||
last_accessed: Option<u64>,
|
||||
) -> Result<()> {
|
||||
use crate::service::media::size;
|
||||
|
||||
let media_info = general_purpose::URL_SAFE_NO_PAD.decode(file_name).unwrap();
|
||||
|
||||
let mxc_dimension_splitter_pos = media_info
|
||||
.iter()
|
||||
.position(|&b| b == 0xff)
|
||||
.ok_or_else(|| Error::BadDatabase("Invalid format of media info from file's name"))?;
|
||||
|
||||
let mxc = utils::string_from_bytes(&media_info[..mxc_dimension_splitter_pos])
|
||||
.map(OwnedMxcUri::from)
|
||||
.map_err(|_| Error::BadDatabase("MXC from file's name is invalid UTF-8."))?;
|
||||
let (server_name, media_id) = mxc
|
||||
.parts()
|
||||
.map_err(|_| Error::BadDatabase("MXC from file's name is invalid."))?;
|
||||
|
||||
let width_height = media_info
|
||||
.get(mxc_dimension_splitter_pos + 1..mxc_dimension_splitter_pos + 9)
|
||||
.ok_or_else(|| Error::BadDatabase("Invalid format of media info from file's name"))?;
|
||||
|
||||
let mut parts = media_info
|
||||
.get(mxc_dimension_splitter_pos + 10..)
|
||||
.ok_or_else(|| Error::BadDatabase("Invalid format of media info from file's name"))?
|
||||
.split(|&b| b == 0xff);
|
||||
|
||||
let content_disposition_bytes = parts.next().ok_or_else(|| {
|
||||
Error::BadDatabase(
|
||||
"Media ID parsed from file's name is invalid: Missing Content Disposition.",
|
||||
)
|
||||
})?;
|
||||
|
||||
let content_disposition = content_disposition_bytes.try_into().unwrap_or_else(|_| {
|
||||
ruma::http_headers::ContentDisposition::new(
|
||||
ruma::http_headers::ContentDispositionType::Inline,
|
||||
)
|
||||
});
|
||||
|
||||
let content_type = parts
|
||||
.next()
|
||||
.map(|bytes| {
|
||||
utils::string_from_bytes(bytes)
|
||||
.map_err(|_| Error::BadDatabase("Content type from file's name is invalid UTF-8."))
|
||||
})
|
||||
.transpose()?;
|
||||
|
||||
let mut path = services()
|
||||
.globals
|
||||
.get_media_folder_only_use_for_migrations();
|
||||
path.push(file_name);
|
||||
|
||||
let mut file = Vec::new();
|
||||
|
||||
tokio::fs::File::open(&path)
|
||||
.await?
|
||||
.read_to_end(&mut file)
|
||||
.await?;
|
||||
let sha256_digest = Sha256::digest(&file);
|
||||
|
||||
let mut zero_zero = 0u32.to_be_bytes().to_vec();
|
||||
zero_zero.extend_from_slice(&0u32.to_be_bytes());
|
||||
|
||||
let mut key = sha256_digest.to_vec();
|
||||
|
||||
let now = utils::secs_since_unix_epoch();
|
||||
let metadata = FilehashMetadata::new_with_times(
|
||||
size(&file)?,
|
||||
creation.unwrap_or(now),
|
||||
last_accessed.unwrap_or(now),
|
||||
);
|
||||
|
||||
db.filehash_metadata.insert(&key, metadata.value())?;
|
||||
|
||||
// If not a thumbnail
|
||||
if width_height == zero_zero {
|
||||
key.extend_from_slice(server_name.as_bytes());
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
|
||||
db.filehash_servername_mediaid.insert(&key, &[])?;
|
||||
|
||||
let mut key = server_name.as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
|
||||
let mut value = sha256_digest.to_vec();
|
||||
value.extend_from_slice(content_disposition.filename.unwrap_or_default().as_bytes());
|
||||
value.push(0xff);
|
||||
value.extend_from_slice(content_type.unwrap_or_default().as_bytes());
|
||||
// To mark as available on unauthenticated endpoints
|
||||
value.push(0xff);
|
||||
|
||||
db.servernamemediaid_metadata.insert(&key, &value)?;
|
||||
} else {
|
||||
key.extend_from_slice(server_name.as_bytes());
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(width_height);
|
||||
|
||||
db.filehash_thumbnailid.insert(&key, &[])?;
|
||||
|
||||
let mut key = server_name.as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(width_height);
|
||||
|
||||
let mut value = sha256_digest.to_vec();
|
||||
value.extend_from_slice(content_disposition.filename.unwrap_or_default().as_bytes());
|
||||
value.push(0xff);
|
||||
value.extend_from_slice(content_type.unwrap_or_default().as_bytes());
|
||||
// To mark as available on unauthenticated endpoints
|
||||
value.push(0xff);
|
||||
|
||||
db.thumbnailid_metadata.insert(&key, &value)?;
|
||||
}
|
||||
|
||||
crate::service::media::create_file(&hex::encode(sha256_digest), &file).await?;
|
||||
tokio::fs::remove_file(path).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sets the emergency password and push rules for the @conduit account in case emergency password is set
|
||||
fn set_emergency_access() -> Result<bool> {
|
||||
let conduit_user = services().globals.server_user();
|
||||
|
|
|
|||
56
src/main.rs
56
src/main.rs
|
|
@ -45,35 +45,55 @@ use tikv_jemallocator::Jemalloc;
|
|||
#[global_allocator]
|
||||
static GLOBAL: Jemalloc = Jemalloc;
|
||||
|
||||
static SUB_TABLES: [&str; 2] = ["well_known", "tls"]; // Not doing `proxy` cause setting that with env vars would be a pain
|
||||
static SUB_TABLES: [&str; 3] = ["well_known", "tls", "media"]; // Not doing `proxy` cause setting that with env vars would be a pain
|
||||
|
||||
// Yeah, I know it's terrible, but since it seems the container users dont want syntax like A[B][C]="...",
|
||||
// this is what we have to deal with. Also see: https://github.com/SergioBenitez/Figment/issues/12#issuecomment-801449465
|
||||
static SUB_SUB_TABLES: [&str; 2] = ["directory_structure", "retention"];
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
clap::parse();
|
||||
|
||||
// Initialize config
|
||||
let raw_config =
|
||||
Figment::new()
|
||||
.merge(
|
||||
Toml::file(Env::var("CONDUIT_CONFIG").expect(
|
||||
let raw_config = Figment::new()
|
||||
.merge(
|
||||
Toml::file(
|
||||
Env::var("CONDUIT_CONFIG").expect(
|
||||
"The CONDUIT_CONFIG env var needs to be set. Example: /etc/conduit.toml",
|
||||
))
|
||||
.nested(),
|
||||
),
|
||||
)
|
||||
.merge(Env::prefixed("CONDUIT_").global().map(|k| {
|
||||
let mut key: Uncased = k.into();
|
||||
.nested(),
|
||||
)
|
||||
.merge(Env::prefixed("CONDUIT_").global().map(|k| {
|
||||
let mut key: Uncased = k.into();
|
||||
|
||||
for table in SUB_TABLES {
|
||||
if k.starts_with(&(table.to_owned() + "_")) {
|
||||
key = Uncased::from(
|
||||
table.to_owned() + "." + k[table.len() + 1..k.len()].as_str(),
|
||||
);
|
||||
break;
|
||||
'outer: for table in SUB_TABLES {
|
||||
if k.starts_with(&(table.to_owned() + "_")) {
|
||||
for sub_table in SUB_SUB_TABLES {
|
||||
if k.starts_with(&(table.to_owned() + "_" + sub_table + "_")) {
|
||||
key = Uncased::from(
|
||||
table.to_owned()
|
||||
+ "."
|
||||
+ sub_table
|
||||
+ "."
|
||||
+ k[table.len() + 1 + sub_table.len() + 1..k.len()].as_str(),
|
||||
);
|
||||
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
key
|
||||
}));
|
||||
key = Uncased::from(
|
||||
table.to_owned() + "." + k[table.len() + 1..k.len()].as_str(),
|
||||
);
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
key
|
||||
}));
|
||||
|
||||
let config = match raw_config.extract::<Config>() {
|
||||
Ok(s) => s,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -7,7 +7,10 @@ use ruma::{
|
|||
|
||||
use crate::api::server_server::DestinationResponse;
|
||||
|
||||
use crate::{config::TurnConfig, services, Config, Error, Result};
|
||||
use crate::{
|
||||
config::{DirectoryStructure, MediaBackendConfig, TurnConfig},
|
||||
services, Config, Error, Result,
|
||||
};
|
||||
use futures_util::FutureExt;
|
||||
use hickory_resolver::TokioAsyncResolver;
|
||||
use hyper_util::client::legacy::connect::dns::{GaiResolver, Name as HyperName};
|
||||
|
|
@ -35,8 +38,6 @@ use tokio::sync::{broadcast, watch::Receiver, Mutex, RwLock, Semaphore};
|
|||
use tower_service::Service as TowerService;
|
||||
use tracing::{error, info};
|
||||
|
||||
use base64::{engine::general_purpose, Engine as _};
|
||||
|
||||
type WellKnownMap = HashMap<OwnedServerName, DestinationResponse>;
|
||||
type TlsNameMap = HashMap<String, (Vec<IpAddr>, u16)>;
|
||||
type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries
|
||||
|
|
@ -227,7 +228,11 @@ impl Service {
|
|||
shutdown: AtomicBool::new(false),
|
||||
};
|
||||
|
||||
fs::create_dir_all(s.get_media_folder())?;
|
||||
// Remove this exception once other media backends are added
|
||||
#[allow(irrefutable_let_patterns)]
|
||||
if let MediaBackendConfig::FileSystem { path, .. } = &s.config.media.backend {
|
||||
fs::create_dir_all(path)?;
|
||||
}
|
||||
|
||||
if !s
|
||||
.supported_room_versions()
|
||||
|
|
@ -349,7 +354,18 @@ impl Service {
|
|||
}
|
||||
|
||||
pub fn turn(&self) -> Option<TurnConfig> {
|
||||
self.config.turn()
|
||||
// We have to clone basically the entire thing on `/turnServers` otherwise
|
||||
self.config.turn.clone()
|
||||
}
|
||||
|
||||
pub fn well_known_server(&self) -> OwnedServerName {
|
||||
// Same as above, but for /.well-known/matrix/server
|
||||
self.config.well_known.server.clone()
|
||||
}
|
||||
|
||||
pub fn well_known_client(&self) -> String {
|
||||
// Same as above, but for /.well-known/matrix/client
|
||||
self.config.well_known.client.clone()
|
||||
}
|
||||
|
||||
pub fn dns_resolver(&self) -> &TokioAsyncResolver {
|
||||
|
|
@ -466,27 +482,32 @@ impl Service {
|
|||
self.db.bump_database_version(new_version)
|
||||
}
|
||||
|
||||
pub fn get_media_folder(&self) -> PathBuf {
|
||||
pub fn get_media_path(
|
||||
&self,
|
||||
media_directory: &str,
|
||||
directory_structure: &DirectoryStructure,
|
||||
sha256_hex: &str,
|
||||
) -> Result<PathBuf> {
|
||||
let mut r = PathBuf::new();
|
||||
r.push(self.config.database_path.clone());
|
||||
r.push("media");
|
||||
r
|
||||
}
|
||||
r.push(media_directory);
|
||||
|
||||
pub fn get_media_file(&self, key: &[u8]) -> PathBuf {
|
||||
let mut r = PathBuf::new();
|
||||
r.push(self.config.database_path.clone());
|
||||
r.push("media");
|
||||
r.push(general_purpose::URL_SAFE_NO_PAD.encode(key));
|
||||
r
|
||||
}
|
||||
if let DirectoryStructure::Deep { length, depth } = directory_structure {
|
||||
let mut filename = sha256_hex;
|
||||
for _ in 0..depth.get() {
|
||||
let (current_path, next) = filename.split_at(length.get().into());
|
||||
filename = next;
|
||||
r.push(current_path);
|
||||
}
|
||||
|
||||
pub fn well_known_server(&self) -> OwnedServerName {
|
||||
self.config.well_known_server()
|
||||
}
|
||||
// Create all directories leading up to file
|
||||
fs::create_dir_all(&r).inspect_err(|e| error!("Error creating leading directories for media with sha256 hash of {sha256_hex}: {e}"))?;
|
||||
|
||||
pub fn well_known_client(&self) -> String {
|
||||
self.config.well_known_client()
|
||||
r.push(filename);
|
||||
} else {
|
||||
r.push(sha256_hex);
|
||||
}
|
||||
|
||||
Ok(r)
|
||||
}
|
||||
|
||||
pub fn shutdown(&self) {
|
||||
|
|
|
|||
|
|
@ -1,22 +1,127 @@
|
|||
use ruma::http_headers::ContentDisposition;
|
||||
use ruma::{OwnedServerName, ServerName, UserId};
|
||||
use sha2::{digest::Output, Sha256};
|
||||
|
||||
use crate::Result;
|
||||
use crate::{config::MediaRetentionConfig, Error, Result};
|
||||
|
||||
use super::{
|
||||
BlockedMediaInfo, DbFileMeta, MediaListItem, MediaQuery, MediaType, ServerNameOrUserId,
|
||||
};
|
||||
|
||||
pub trait Data: Send + Sync {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn create_file_metadata(
|
||||
&self,
|
||||
mxc: String,
|
||||
width: u32,
|
||||
height: u32,
|
||||
content_disposition: &ContentDisposition,
|
||||
sha256_digest: Output<Sha256>,
|
||||
file_size: u64,
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
filename: Option<&str>,
|
||||
content_type: Option<&str>,
|
||||
) -> Result<Vec<u8>>;
|
||||
user_id: Option<&UserId>,
|
||||
is_blocked_filehash: bool,
|
||||
) -> Result<()>;
|
||||
|
||||
/// Returns content_disposition, content_type and the metadata key.
|
||||
fn search_file_metadata(
|
||||
fn search_file_metadata(&self, servername: &ServerName, media_id: &str) -> Result<DbFileMeta>;
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn create_thumbnail_metadata(
|
||||
&self,
|
||||
mxc: String,
|
||||
sha256_digest: Output<Sha256>,
|
||||
file_size: u64,
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) -> Result<(ContentDisposition, Option<String>, Vec<u8>)>;
|
||||
filename: Option<&str>,
|
||||
content_type: Option<&str>,
|
||||
) -> Result<()>;
|
||||
|
||||
// Returns the sha256 hash, filename and content_type and whether the media should be accessible via
|
||||
/// unauthenticated endpoints.
|
||||
fn search_thumbnail_metadata(
|
||||
&self,
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) -> Result<DbFileMeta>;
|
||||
|
||||
fn query(&self, server_name: &ServerName, media_id: &str) -> Result<MediaQuery>;
|
||||
|
||||
fn purge_and_get_hashes(
|
||||
&self,
|
||||
media: &[(OwnedServerName, String)],
|
||||
force_filehash: bool,
|
||||
) -> Vec<Result<String>>;
|
||||
|
||||
fn purge_and_get_hashes_from_user(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
force_filehash: bool,
|
||||
after: Option<u64>,
|
||||
) -> Vec<Result<String>>;
|
||||
|
||||
fn purge_and_get_hashes_from_server(
|
||||
&self,
|
||||
server_name: &ServerName,
|
||||
force_filehash: bool,
|
||||
after: Option<u64>,
|
||||
) -> Vec<Result<String>>;
|
||||
|
||||
fn is_blocked(&self, server_name: &ServerName, media_id: &str) -> Result<bool>;
|
||||
|
||||
fn block(
|
||||
&self,
|
||||
media: &[(OwnedServerName, String)],
|
||||
unix_secs: u64,
|
||||
reason: Option<String>,
|
||||
) -> Vec<Error>;
|
||||
|
||||
fn block_from_user(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
now: u64,
|
||||
reason: &str,
|
||||
after: Option<u64>,
|
||||
) -> Vec<Error>;
|
||||
|
||||
fn unblock(&self, media: &[(OwnedServerName, String)]) -> Vec<Error>;
|
||||
|
||||
fn list(
|
||||
&self,
|
||||
server_name_or_user_id: Option<ServerNameOrUserId>,
|
||||
include_thumbnails: bool,
|
||||
content_type: Option<&str>,
|
||||
before: Option<u64>,
|
||||
after: Option<u64>,
|
||||
) -> Result<Vec<MediaListItem>>;
|
||||
|
||||
/// Returns a Vec of:
|
||||
/// - The server the media is from
|
||||
/// - The media id
|
||||
/// - The time it was blocked, in unix seconds
|
||||
/// - The optional reason why it was blocked
|
||||
fn list_blocked(&self) -> Vec<Result<BlockedMediaInfo>>;
|
||||
|
||||
fn is_blocked_filehash(&self, sha256_digest: &[u8]) -> Result<bool>;
|
||||
|
||||
/// Gets the files that need to be deleted from the media backend in order to meet the `space`
|
||||
/// requirements, as specified in the retention config. Calling this also causes those files'
|
||||
/// metadata to be deleted from the database.
|
||||
fn files_to_delete(
|
||||
&self,
|
||||
sha256_digest: &[u8],
|
||||
retention: &MediaRetentionConfig,
|
||||
media_type: MediaType,
|
||||
new_size: u64,
|
||||
) -> Result<Vec<Result<String>>>;
|
||||
|
||||
/// Gets the files that need to be deleted from the media backend in order to meet the
|
||||
/// time-based requirements (`created` and `accessed`), as specified in the retention config.
|
||||
/// Calling this also causes those files' metadata to be deleted from the database.
|
||||
fn cleanup_time_retention(&self, retention: &MediaRetentionConfig) -> Vec<Result<String>>;
|
||||
|
||||
fn update_last_accessed(&self, server_name: &ServerName, media_id: &str) -> Result<()>;
|
||||
|
||||
fn update_last_accessed_filehash(&self, sha256_digest: &[u8]) -> Result<()>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,97 +1,253 @@
|
|||
mod data;
|
||||
use std::io::Cursor;
|
||||
use std::{fs, io::Cursor, sync::Arc};
|
||||
|
||||
pub use data::Data;
|
||||
use ruma::{
|
||||
api::client::error::ErrorKind,
|
||||
api::client::{error::ErrorKind, media::is_safe_inline_content_type},
|
||||
http_headers::{ContentDisposition, ContentDispositionType},
|
||||
OwnedServerName, ServerName, UserId,
|
||||
};
|
||||
use sha2::{digest::Output, Digest, Sha256};
|
||||
use tracing::{error, info};
|
||||
|
||||
use crate::{services, Result};
|
||||
use crate::{
|
||||
config::{DirectoryStructure, MediaBackendConfig},
|
||||
services, utils, Error, Result,
|
||||
};
|
||||
use image::imageops::FilterType;
|
||||
|
||||
pub struct DbFileMeta {
|
||||
pub sha256_digest: Vec<u8>,
|
||||
pub filename: Option<String>,
|
||||
pub content_type: Option<String>,
|
||||
pub unauthenticated_access_permitted: bool,
|
||||
}
|
||||
|
||||
use tokio::{
|
||||
fs::File,
|
||||
io::{AsyncReadExt, AsyncWriteExt, BufReader},
|
||||
io::{AsyncReadExt, AsyncWriteExt},
|
||||
};
|
||||
|
||||
pub struct MediaQuery {
|
||||
pub is_blocked: bool,
|
||||
pub source_file: Option<MediaQueryFileInfo>,
|
||||
pub thumbnails: Vec<MediaQueryThumbInfo>,
|
||||
}
|
||||
|
||||
pub struct MediaQueryFileInfo {
|
||||
pub uploader_localpart: Option<String>,
|
||||
pub sha256_hex: String,
|
||||
pub filename: Option<String>,
|
||||
pub content_type: Option<String>,
|
||||
pub unauthenticated_access_permitted: bool,
|
||||
pub is_blocked_via_filehash: bool,
|
||||
pub file_info: Option<FileInfo>,
|
||||
}
|
||||
|
||||
pub struct MediaQueryThumbInfo {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub sha256_hex: String,
|
||||
pub filename: Option<String>,
|
||||
pub content_type: Option<String>,
|
||||
pub unauthenticated_access_permitted: bool,
|
||||
pub is_blocked_via_filehash: bool,
|
||||
pub file_info: Option<FileInfo>,
|
||||
}
|
||||
|
||||
pub struct FileInfo {
|
||||
pub creation: u64,
|
||||
pub last_access: u64,
|
||||
pub size: u64,
|
||||
}
|
||||
|
||||
pub struct MediaListItem {
|
||||
pub server_name: OwnedServerName,
|
||||
pub media_id: String,
|
||||
pub uploader_localpart: Option<String>,
|
||||
pub content_type: Option<String>,
|
||||
pub filename: Option<String>,
|
||||
pub dimensions: Option<(u32, u32)>,
|
||||
pub size: u64,
|
||||
pub creation: u64,
|
||||
}
|
||||
|
||||
pub enum ServerNameOrUserId {
|
||||
ServerName(Box<ServerName>),
|
||||
UserId(Box<UserId>),
|
||||
}
|
||||
|
||||
pub struct FileMeta {
|
||||
pub content_disposition: ContentDisposition,
|
||||
pub content_type: Option<String>,
|
||||
pub file: Vec<u8>,
|
||||
}
|
||||
|
||||
pub enum MediaType {
|
||||
LocalMedia { thumbnail: bool },
|
||||
RemoteMedia { thumbnail: bool },
|
||||
}
|
||||
|
||||
impl MediaType {
|
||||
pub fn new(server_name: &ServerName, thumbnail: bool) -> Self {
|
||||
if server_name == services().globals.server_name() {
|
||||
Self::LocalMedia { thumbnail }
|
||||
} else {
|
||||
Self::RemoteMedia { thumbnail }
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_thumb(&self) -> bool {
|
||||
match self {
|
||||
MediaType::LocalMedia { thumbnail } | MediaType::RemoteMedia { thumbnail } => {
|
||||
*thumbnail
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Service {
|
||||
pub db: &'static dyn Data,
|
||||
}
|
||||
|
||||
pub struct BlockedMediaInfo {
|
||||
pub server_name: OwnedServerName,
|
||||
pub media_id: String,
|
||||
pub unix_secs: u64,
|
||||
pub reason: Option<String>,
|
||||
pub sha256_hex: Option<String>,
|
||||
}
|
||||
|
||||
impl Service {
|
||||
pub fn start_time_retention_checker(self: &Arc<Self>) {
|
||||
let self2 = Arc::clone(self);
|
||||
if let Some(cleanup_interval) = services().globals.config.media.retention.cleanup_interval()
|
||||
{
|
||||
tokio::spawn(async move {
|
||||
let mut i = cleanup_interval;
|
||||
loop {
|
||||
i.tick().await;
|
||||
let _ = self2.try_purge_time_retention().await;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async fn try_purge_time_retention(&self) -> Result<()> {
|
||||
info!("Checking if any media should be deleted due to time-based retention policies");
|
||||
let files = self
|
||||
.db
|
||||
.cleanup_time_retention(&services().globals.config.media.retention);
|
||||
|
||||
let count = files.iter().filter(|res| res.is_ok()).count();
|
||||
info!("Found {count} media files to delete");
|
||||
|
||||
purge_files(files);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Uploads a file.
|
||||
pub async fn create(
|
||||
&self,
|
||||
mxc: String,
|
||||
content_disposition: Option<ContentDisposition>,
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
filename: Option<&str>,
|
||||
content_type: Option<&str>,
|
||||
file: &[u8],
|
||||
user_id: Option<&UserId>,
|
||||
) -> Result<()> {
|
||||
let content_disposition =
|
||||
content_disposition.unwrap_or(ContentDisposition::new(ContentDispositionType::Inline));
|
||||
let (sha256_digest, sha256_hex) = generate_digests(file);
|
||||
|
||||
// Width, Height = 0 if it's not a thumbnail
|
||||
let key = self
|
||||
.db
|
||||
.create_file_metadata(mxc, 0, 0, &content_disposition, content_type)?;
|
||||
for error in self.clear_required_space(
|
||||
&sha256_digest,
|
||||
MediaType::new(servername, false),
|
||||
size(file)?,
|
||||
)? {
|
||||
error!(
|
||||
"Error deleting file to clear space when downloading/creating new media file: {error}"
|
||||
)
|
||||
}
|
||||
|
||||
let path = services().globals.get_media_file(&key);
|
||||
let mut f = File::create(path).await?;
|
||||
f.write_all(file).await?;
|
||||
Ok(())
|
||||
self.db.create_file_metadata(
|
||||
sha256_digest,
|
||||
size(file)?,
|
||||
servername,
|
||||
media_id,
|
||||
filename,
|
||||
content_type,
|
||||
user_id,
|
||||
self.db.is_blocked_filehash(&sha256_digest)?,
|
||||
)?;
|
||||
|
||||
if !self.db.is_blocked_filehash(&sha256_digest)? {
|
||||
create_file(&sha256_hex, file).await
|
||||
} else if user_id.is_none() {
|
||||
Err(Error::BadRequest(ErrorKind::NotFound, "Media not found."))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Uploads or replaces a file thumbnail.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn upload_thumbnail(
|
||||
&self,
|
||||
mxc: String,
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
filename: Option<&str>,
|
||||
content_type: Option<&str>,
|
||||
width: u32,
|
||||
height: u32,
|
||||
file: &[u8],
|
||||
) -> Result<()> {
|
||||
let key = self.db.create_file_metadata(
|
||||
mxc,
|
||||
let (sha256_digest, sha256_hex) = generate_digests(file);
|
||||
|
||||
self.clear_required_space(
|
||||
&sha256_digest,
|
||||
MediaType::new(servername, true),
|
||||
size(file)?,
|
||||
)?;
|
||||
|
||||
self.db.create_thumbnail_metadata(
|
||||
sha256_digest,
|
||||
size(file)?,
|
||||
servername,
|
||||
media_id,
|
||||
width,
|
||||
height,
|
||||
&ContentDisposition::new(ContentDispositionType::Inline),
|
||||
filename,
|
||||
content_type,
|
||||
)?;
|
||||
|
||||
let path = services().globals.get_media_file(&key);
|
||||
let mut f = File::create(path).await?;
|
||||
f.write_all(file).await?;
|
||||
|
||||
Ok(())
|
||||
create_file(&sha256_hex, file).await
|
||||
}
|
||||
|
||||
/// Downloads a file.
|
||||
pub async fn get(&self, mxc: String) -> Result<Option<FileMeta>> {
|
||||
if let Ok((content_disposition, content_type, key)) =
|
||||
self.db.search_file_metadata(mxc, 0, 0)
|
||||
{
|
||||
let path = services().globals.get_media_file(&key);
|
||||
let mut file = Vec::new();
|
||||
BufReader::new(File::open(path).await?)
|
||||
.read_to_end(&mut file)
|
||||
.await?;
|
||||
/// Fetches a local file and it's metadata
|
||||
pub async fn get(
|
||||
&self,
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
authenticated: bool,
|
||||
) -> Result<Option<FileMeta>> {
|
||||
let DbFileMeta {
|
||||
sha256_digest,
|
||||
filename,
|
||||
content_type,
|
||||
unauthenticated_access_permitted,
|
||||
} = self.db.search_file_metadata(servername, media_id)?;
|
||||
|
||||
Ok(Some(FileMeta {
|
||||
content_disposition,
|
||||
content_type,
|
||||
file,
|
||||
}))
|
||||
} else {
|
||||
Ok(None)
|
||||
if !(authenticated || unauthenticated_access_permitted) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let file = self.get_file(&sha256_digest, None).await?;
|
||||
|
||||
Ok(Some(FileMeta {
|
||||
content_disposition: content_disposition(filename, &content_type),
|
||||
content_type,
|
||||
file,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Returns width, height of the thumbnail and whether it should be cropped. Returns None when
|
||||
|
|
@ -119,117 +275,462 @@ impl Service {
|
|||
/// For width,height <= 96 the server uses another thumbnailing algorithm which crops the image afterwards.
|
||||
pub async fn get_thumbnail(
|
||||
&self,
|
||||
mxc: String,
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
width: u32,
|
||||
height: u32,
|
||||
authenticated: bool,
|
||||
) -> Result<Option<FileMeta>> {
|
||||
let (width, height, crop) = self
|
||||
.thumbnail_properties(width, height)
|
||||
.unwrap_or((0, 0, false)); // 0, 0 because that's the original file
|
||||
|
||||
if let Ok((content_disposition, content_type, key)) =
|
||||
self.db.search_file_metadata(mxc.clone(), width, height)
|
||||
{
|
||||
// Using saved thumbnail
|
||||
let path = services().globals.get_media_file(&key);
|
||||
let mut file = Vec::new();
|
||||
File::open(path).await?.read_to_end(&mut file).await?;
|
||||
|
||||
Ok(Some(FileMeta {
|
||||
content_disposition,
|
||||
if let Some((width, height, crop)) = self.thumbnail_properties(width, height) {
|
||||
if let Ok(DbFileMeta {
|
||||
sha256_digest,
|
||||
filename,
|
||||
content_type,
|
||||
file: file.to_vec(),
|
||||
}))
|
||||
} else if let Ok((content_disposition, content_type, key)) =
|
||||
self.db.search_file_metadata(mxc.clone(), 0, 0)
|
||||
{
|
||||
// Generate a thumbnail
|
||||
let path = services().globals.get_media_file(&key);
|
||||
let mut file = Vec::new();
|
||||
File::open(path).await?.read_to_end(&mut file).await?;
|
||||
|
||||
if let Ok(image) = image::load_from_memory(&file) {
|
||||
let original_width = image.width();
|
||||
let original_height = image.height();
|
||||
if width > original_width || height > original_height {
|
||||
return Ok(Some(FileMeta {
|
||||
content_disposition,
|
||||
content_type,
|
||||
file: file.to_vec(),
|
||||
}));
|
||||
unauthenticated_access_permitted,
|
||||
}) = self
|
||||
.db
|
||||
.search_thumbnail_metadata(servername, media_id, width, height)
|
||||
{
|
||||
if !(authenticated || unauthenticated_access_permitted) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let thumbnail = if crop {
|
||||
image.resize_to_fill(width, height, FilterType::CatmullRom)
|
||||
} else {
|
||||
let (exact_width, exact_height) = {
|
||||
// Copied from image::dynimage::resize_dimensions
|
||||
let ratio = u64::from(original_width) * u64::from(height);
|
||||
let nratio = u64::from(width) * u64::from(original_height);
|
||||
|
||||
let use_width = nratio <= ratio;
|
||||
let intermediate = if use_width {
|
||||
u64::from(original_height) * u64::from(width)
|
||||
/ u64::from(original_width)
|
||||
} else {
|
||||
u64::from(original_width) * u64::from(height)
|
||||
/ u64::from(original_height)
|
||||
};
|
||||
if use_width {
|
||||
if intermediate <= u64::from(u32::MAX) {
|
||||
(width, intermediate as u32)
|
||||
} else {
|
||||
(
|
||||
(u64::from(width) * u64::from(u32::MAX) / intermediate) as u32,
|
||||
u32::MAX,
|
||||
)
|
||||
}
|
||||
} else if intermediate <= u64::from(u32::MAX) {
|
||||
(intermediate as u32, height)
|
||||
} else {
|
||||
(
|
||||
u32::MAX,
|
||||
(u64::from(height) * u64::from(u32::MAX) / intermediate) as u32,
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
image.thumbnail_exact(exact_width, exact_height)
|
||||
};
|
||||
|
||||
let mut thumbnail_bytes = Vec::new();
|
||||
thumbnail.write_to(
|
||||
&mut Cursor::new(&mut thumbnail_bytes),
|
||||
image::ImageFormat::Png,
|
||||
)?;
|
||||
|
||||
// Save thumbnail in database so we don't have to generate it again next time
|
||||
let thumbnail_key = self.db.create_file_metadata(
|
||||
mxc,
|
||||
width,
|
||||
height,
|
||||
&content_disposition,
|
||||
content_type.as_deref(),
|
||||
)?;
|
||||
|
||||
let path = services().globals.get_media_file(&thumbnail_key);
|
||||
let mut f = File::create(path).await?;
|
||||
f.write_all(&thumbnail_bytes).await?;
|
||||
// Using saved thumbnail
|
||||
let file = self
|
||||
.get_file(&sha256_digest, Some((servername, media_id)))
|
||||
.await?;
|
||||
|
||||
Ok(Some(FileMeta {
|
||||
content_disposition,
|
||||
content_disposition: content_disposition(filename, &content_type),
|
||||
content_type,
|
||||
file: thumbnail_bytes.to_vec(),
|
||||
file,
|
||||
}))
|
||||
} else if !authenticated {
|
||||
return Ok(None);
|
||||
} else if let Ok(DbFileMeta {
|
||||
sha256_digest,
|
||||
filename,
|
||||
content_type,
|
||||
unauthenticated_access_permitted,
|
||||
}) = self.db.search_file_metadata(servername, media_id)
|
||||
{
|
||||
if !(authenticated || unauthenticated_access_permitted) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let content_disposition = content_disposition(filename.clone(), &content_type);
|
||||
// Generate a thumbnail
|
||||
let file = self.get_file(&sha256_digest, None).await?;
|
||||
|
||||
if let Ok(image) = image::load_from_memory(&file) {
|
||||
let original_width = image.width();
|
||||
let original_height = image.height();
|
||||
if width > original_width || height > original_height {
|
||||
return Ok(Some(FileMeta {
|
||||
content_disposition,
|
||||
content_type,
|
||||
file,
|
||||
}));
|
||||
}
|
||||
|
||||
let thumbnail = if crop {
|
||||
image.resize_to_fill(width, height, FilterType::CatmullRom)
|
||||
} else {
|
||||
let (exact_width, exact_height) = {
|
||||
// Copied from image::dynimage::resize_dimensions
|
||||
let ratio = u64::from(original_width) * u64::from(height);
|
||||
let nratio = u64::from(width) * u64::from(original_height);
|
||||
|
||||
let use_width = nratio <= ratio;
|
||||
let intermediate = if use_width {
|
||||
u64::from(original_height) * u64::from(width)
|
||||
/ u64::from(original_width)
|
||||
} else {
|
||||
u64::from(original_width) * u64::from(height)
|
||||
/ u64::from(original_height)
|
||||
};
|
||||
if use_width {
|
||||
if intermediate <= u64::from(u32::MAX) {
|
||||
(width, intermediate as u32)
|
||||
} else {
|
||||
(
|
||||
(u64::from(width) * u64::from(u32::MAX) / intermediate)
|
||||
as u32,
|
||||
u32::MAX,
|
||||
)
|
||||
}
|
||||
} else if intermediate <= u64::from(u32::MAX) {
|
||||
(intermediate as u32, height)
|
||||
} else {
|
||||
(
|
||||
u32::MAX,
|
||||
(u64::from(height) * u64::from(u32::MAX) / intermediate) as u32,
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
image.thumbnail_exact(exact_width, exact_height)
|
||||
};
|
||||
|
||||
let mut thumbnail_bytes = Vec::new();
|
||||
thumbnail.write_to(
|
||||
&mut Cursor::new(&mut thumbnail_bytes),
|
||||
image::ImageFormat::Png,
|
||||
)?;
|
||||
|
||||
// Save thumbnail in database so we don't have to generate it again next time
|
||||
self.upload_thumbnail(
|
||||
servername,
|
||||
media_id,
|
||||
filename.as_deref(),
|
||||
content_type.as_deref(),
|
||||
width,
|
||||
height,
|
||||
&thumbnail_bytes,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(Some(FileMeta {
|
||||
content_disposition,
|
||||
content_type,
|
||||
file: thumbnail_bytes,
|
||||
}))
|
||||
} else {
|
||||
// Couldn't parse file to generate thumbnail, likely not an image
|
||||
Err(Error::BadRequest(
|
||||
ErrorKind::Unknown,
|
||||
"Unable to generate thumbnail for the requested content (likely is not an image)",
|
||||
))
|
||||
}
|
||||
} else {
|
||||
// Couldn't parse file to generate thumbnail, likely not an image
|
||||
return Err(crate::Error::BadRequest(
|
||||
ErrorKind::Unknown,
|
||||
"Unable to generate thumbnail for the requested content (likely is not an image)",
|
||||
));
|
||||
Ok(None)
|
||||
}
|
||||
} else {
|
||||
Ok(None)
|
||||
// Using full-sized file
|
||||
let Ok(DbFileMeta {
|
||||
sha256_digest,
|
||||
filename,
|
||||
content_type,
|
||||
unauthenticated_access_permitted,
|
||||
}) = self.db.search_file_metadata(servername, media_id)
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
if !(authenticated || unauthenticated_access_permitted) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let file = self.get_file(&sha256_digest, None).await?;
|
||||
|
||||
Ok(Some(FileMeta {
|
||||
content_disposition: content_disposition(filename, &content_type),
|
||||
content_type,
|
||||
file,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns information about the queried media
|
||||
pub fn query(&self, server_name: &ServerName, media_id: &str) -> Result<MediaQuery> {
|
||||
self.db.query(server_name, media_id)
|
||||
}
|
||||
|
||||
/// Purges all of the specified media.
|
||||
///
|
||||
/// If `force_filehash` is true, all media and/or thumbnails which share sha256 content hashes
|
||||
/// with the purged media will also be purged, meaning that the media is guaranteed to be deleted
|
||||
/// from the media backend. Otherwise, it will be deleted if only the media IDs requested to be
|
||||
/// purged have that sha256 hash.
|
||||
///
|
||||
/// Returns errors for all the files that were failed to be deleted, if any.
|
||||
pub fn purge(&self, media: &[(OwnedServerName, String)], force_filehash: bool) -> Vec<Error> {
|
||||
let hashes = self.db.purge_and_get_hashes(media, force_filehash);
|
||||
|
||||
purge_files(hashes)
|
||||
}
|
||||
|
||||
/// Purges all (past a certain time in unix seconds, if specified) media
|
||||
/// sent by a user.
|
||||
///
|
||||
/// If `force_filehash` is true, all media and/or thumbnails which share sha256 content hashes
|
||||
/// with the purged media will also be purged, meaning that the media is guaranteed to be deleted
|
||||
/// from the media backend. Otherwise, it will be deleted if only the media IDs requested to be
|
||||
/// purged have that sha256 hash.
|
||||
///
|
||||
/// Returns errors for all the files that were failed to be deleted, if any.
|
||||
///
|
||||
/// Note: it only currently works for local users, as we cannot determine who
|
||||
/// exactly uploaded the file when it comes to remove users.
|
||||
pub fn purge_from_user(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
force_filehash: bool,
|
||||
after: Option<u64>,
|
||||
) -> Vec<Error> {
|
||||
let hashes = self
|
||||
.db
|
||||
.purge_and_get_hashes_from_user(user_id, force_filehash, after);
|
||||
|
||||
purge_files(hashes)
|
||||
}
|
||||
|
||||
/// Purges all (past a certain time in unix seconds, if specified) media
|
||||
/// obtained from the specified server (due to the MXC URI).
|
||||
///
|
||||
/// If `force_filehash` is true, all media and/or thumbnails which share sha256 content hashes
|
||||
/// with the purged media will also be purged, meaning that the media is guaranteed to be deleted
|
||||
/// from the media backend. Otherwise, it will be deleted if only the media IDs requested to be
|
||||
/// purged have that sha256 hash.
|
||||
///
|
||||
/// Returns errors for all the files that were failed to be deleted, if any.
|
||||
pub fn purge_from_server(
|
||||
&self,
|
||||
server_name: &ServerName,
|
||||
force_filehash: bool,
|
||||
after: Option<u64>,
|
||||
) -> Vec<Error> {
|
||||
let hashes = self
|
||||
.db
|
||||
.purge_and_get_hashes_from_server(server_name, force_filehash, after);
|
||||
|
||||
purge_files(hashes)
|
||||
}
|
||||
|
||||
/// Checks whether the media has been blocked by administrators, returning either
|
||||
/// a database error, or a not found error if it is blocked
|
||||
pub fn check_blocked(&self, server_name: &ServerName, media_id: &str) -> Result<()> {
|
||||
if self.db.is_blocked(server_name, media_id)? {
|
||||
Err(Error::BadRequest(ErrorKind::NotFound, "Media not found."))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Marks the specified media as blocked, preventing them from being accessed
|
||||
pub fn block(&self, media: &[(OwnedServerName, String)], reason: Option<String>) -> Vec<Error> {
|
||||
let now = utils::secs_since_unix_epoch();
|
||||
|
||||
self.db.block(media, now, reason)
|
||||
}
|
||||
|
||||
/// Marks the media uploaded by a local user as blocked, preventing it from being accessed
|
||||
pub fn block_from_user(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
reason: &str,
|
||||
after: Option<u64>,
|
||||
) -> Vec<Error> {
|
||||
let now = utils::secs_since_unix_epoch();
|
||||
|
||||
self.db.block_from_user(user_id, now, reason, after)
|
||||
}
|
||||
|
||||
/// Unblocks the specified media, allowing them from being accessed again
|
||||
pub fn unblock(&self, media: &[(OwnedServerName, String)]) -> Vec<Error> {
|
||||
self.db.unblock(media)
|
||||
}
|
||||
|
||||
/// Returns a list of all the stored media, applying all the given filters to the results
|
||||
pub fn list(
|
||||
&self,
|
||||
server_name_or_user_id: Option<ServerNameOrUserId>,
|
||||
include_thumbnails: bool,
|
||||
content_type: Option<&str>,
|
||||
before: Option<u64>,
|
||||
after: Option<u64>,
|
||||
) -> Result<Vec<MediaListItem>> {
|
||||
self.db.list(
|
||||
server_name_or_user_id,
|
||||
include_thumbnails,
|
||||
content_type,
|
||||
before,
|
||||
after,
|
||||
)
|
||||
}
|
||||
|
||||
/// Returns a Vec of:
|
||||
/// - The server the media is from
|
||||
/// - The media id
|
||||
/// - The time it was blocked, in unix seconds
|
||||
/// - The optional reason why it was blocked
|
||||
pub fn list_blocked(&self) -> Vec<Result<BlockedMediaInfo>> {
|
||||
self.db.list_blocked()
|
||||
}
|
||||
|
||||
pub fn clear_required_space(
|
||||
&self,
|
||||
sha256_digest: &[u8],
|
||||
media_type: MediaType,
|
||||
new_size: u64,
|
||||
) -> Result<Vec<Error>> {
|
||||
let files = self.db.files_to_delete(
|
||||
sha256_digest,
|
||||
&services().globals.config.media.retention,
|
||||
media_type,
|
||||
new_size,
|
||||
)?;
|
||||
|
||||
let count = files.iter().filter(|r| r.is_ok()).count();
|
||||
|
||||
if count != 0 {
|
||||
info!("Deleting {} files to clear space for new media file", count);
|
||||
}
|
||||
|
||||
Ok(purge_files(files))
|
||||
}
|
||||
|
||||
/// Fetches the file from the configured media backend, as well as updating the "last accessed"
|
||||
/// part of the metadata of the file
|
||||
///
|
||||
/// If specified, the original file will also have it's last accessed time updated, if present
|
||||
/// (use when accessing thumbnails)
|
||||
async fn get_file(
|
||||
&self,
|
||||
sha256_digest: &[u8],
|
||||
original_file_id: Option<(&ServerName, &str)>,
|
||||
) -> Result<Vec<u8>> {
|
||||
let file = match &services().globals.config.media.backend {
|
||||
MediaBackendConfig::FileSystem {
|
||||
path,
|
||||
directory_structure,
|
||||
} => {
|
||||
let path = services().globals.get_media_path(
|
||||
path,
|
||||
directory_structure,
|
||||
&hex::encode(sha256_digest),
|
||||
)?;
|
||||
|
||||
let mut file = Vec::new();
|
||||
File::open(path).await?.read_to_end(&mut file).await?;
|
||||
|
||||
file
|
||||
}
|
||||
};
|
||||
|
||||
if let Some((server_name, media_id)) = original_file_id {
|
||||
self.db.update_last_accessed(server_name, media_id)?;
|
||||
}
|
||||
|
||||
self.db
|
||||
.update_last_accessed_filehash(sha256_digest)
|
||||
.map(|_| file)
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates the media file, using the configured media backend
|
||||
///
|
||||
/// Note: this function does NOT set the metadata related to the file
|
||||
pub async fn create_file(sha256_hex: &str, file: &[u8]) -> Result<()> {
|
||||
match &services().globals.config.media.backend {
|
||||
MediaBackendConfig::FileSystem {
|
||||
path,
|
||||
directory_structure,
|
||||
} => {
|
||||
let path = services()
|
||||
.globals
|
||||
.get_media_path(path, directory_structure, sha256_hex)?;
|
||||
|
||||
let mut f = File::create(path).await?;
|
||||
f.write_all(file).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Purges the given files from the media backend
|
||||
/// Returns a `Vec` of errors that occurred when attempting to delete the files
|
||||
///
|
||||
/// Note: this does NOT remove the related metadata from the database
|
||||
fn purge_files(hashes: Vec<Result<String>>) -> Vec<Error> {
|
||||
hashes
|
||||
.into_iter()
|
||||
.map(|hash| match hash {
|
||||
Ok(v) => delete_file(&v),
|
||||
Err(e) => Err(e),
|
||||
})
|
||||
.filter_map(|r| if let Err(e) = r { Some(e) } else { None })
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Deletes the given file from the media backend
|
||||
///
|
||||
/// Note: this does NOT remove the related metadata from the database
|
||||
fn delete_file(sha256_hex: &str) -> Result<()> {
|
||||
match &services().globals.config.media.backend {
|
||||
MediaBackendConfig::FileSystem {
|
||||
path,
|
||||
directory_structure,
|
||||
} => {
|
||||
let mut path =
|
||||
services()
|
||||
.globals
|
||||
.get_media_path(path, directory_structure, sha256_hex)?;
|
||||
|
||||
if let Err(e) = fs::remove_file(&path) {
|
||||
// Multiple files with the same filehash might be requseted to be deleted
|
||||
if e.kind() != std::io::ErrorKind::NotFound {
|
||||
error!("Error removing media from filesystem: {e}");
|
||||
Err(e)?;
|
||||
}
|
||||
}
|
||||
|
||||
if let DirectoryStructure::Deep { length: _, depth } = directory_structure {
|
||||
let mut depth = depth.get();
|
||||
|
||||
while depth > 0 {
|
||||
// Here at the start so that the first time, the file gets removed from the path
|
||||
path.pop();
|
||||
|
||||
if let Err(e) = fs::remove_dir(&path) {
|
||||
if e.kind() == std::io::ErrorKind::DirectoryNotEmpty {
|
||||
break;
|
||||
} else {
|
||||
error!("Error removing empty media directories: {e}");
|
||||
Err(e)?;
|
||||
}
|
||||
}
|
||||
|
||||
depth -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Creates a content disposition with the given `filename`, using the `content_type` to determine whether
|
||||
/// the disposition should be `inline` or `attachment`
|
||||
fn content_disposition(
|
||||
filename: Option<String>,
|
||||
content_type: &Option<String>,
|
||||
) -> ContentDisposition {
|
||||
ContentDisposition::new(
|
||||
if content_type
|
||||
.as_deref()
|
||||
.is_some_and(is_safe_inline_content_type)
|
||||
{
|
||||
ContentDispositionType::Inline
|
||||
} else {
|
||||
ContentDispositionType::Attachment
|
||||
},
|
||||
)
|
||||
.with_filename(filename)
|
||||
}
|
||||
|
||||
/// Returns sha256 digests of the file, in raw (Vec) and hex form respectively
|
||||
fn generate_digests(file: &[u8]) -> (Output<Sha256>, String) {
|
||||
let sha256_digest = Sha256::digest(file);
|
||||
let hex_sha256 = hex::encode(sha256_digest);
|
||||
|
||||
(sha256_digest, hex_sha256)
|
||||
}
|
||||
|
||||
/// Get's the file size, is bytes, as u64, returning an error if the file size is larger
|
||||
/// than a u64 (which is far too big to be reasonably uploaded in the first place anyways)
|
||||
pub fn size(file: &[u8]) -> Result<u64> {
|
||||
u64::try_from(file.len())
|
||||
.map_err(|_| Error::BadRequest(ErrorKind::TooLarge, "File is too large"))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ pub struct Services {
|
|||
pub admin: Arc<admin::Service>,
|
||||
pub globals: globals::Service,
|
||||
pub key_backups: key_backups::Service,
|
||||
pub media: media::Service,
|
||||
pub media: Arc<media::Service>,
|
||||
pub sending: Arc<sending::Service>,
|
||||
}
|
||||
|
||||
|
|
@ -119,7 +119,7 @@ impl Services {
|
|||
account_data: account_data::Service { db },
|
||||
admin: admin::Service::build(),
|
||||
key_backups: key_backups::Service { db },
|
||||
media: media::Service { db },
|
||||
media: Arc::new(media::Service { db }),
|
||||
sending: sending::Service::build(db, &config),
|
||||
|
||||
globals: globals::Service::load(db, config)?,
|
||||
|
|
|
|||
|
|
@ -18,6 +18,13 @@ pub fn millis_since_unix_epoch() -> u64 {
|
|||
.as_millis() as u64
|
||||
}
|
||||
|
||||
pub fn secs_since_unix_epoch() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("time is valid")
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
pub fn increment(old: Option<&[u8]>) -> Option<Vec<u8>> {
|
||||
let number = match old.map(|bytes| bytes.try_into()) {
|
||||
Some(Ok(bytes)) => {
|
||||
|
|
@ -127,7 +134,7 @@ pub fn deserialize_from_str<
|
|||
deserializer: D,
|
||||
) -> Result<T, D::Error> {
|
||||
struct Visitor<T: FromStr<Err = E>, E>(std::marker::PhantomData<T>);
|
||||
impl<'de, T: FromStr<Err = Err>, Err: fmt::Display> serde::de::Visitor<'de> for Visitor<T, Err> {
|
||||
impl<T: FromStr<Err = Err>, Err: fmt::Display> serde::de::Visitor<'_> for Visitor<T, Err> {
|
||||
type Value = T;
|
||||
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(formatter, "a parsable string")
|
||||
|
|
@ -149,7 +156,7 @@ pub fn deserialize_from_str<
|
|||
/// string when passed to a format string.
|
||||
pub struct HtmlEscape<'a>(pub &'a str);
|
||||
|
||||
impl<'a> fmt::Display for HtmlEscape<'a> {
|
||||
impl fmt::Display for HtmlEscape<'_> {
|
||||
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
// Because the internet is always right, turns out there's not that many
|
||||
// characters to escape: http://stackoverflow.com/questions/7381974
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue