mirror of
https://gitlab.com/famedly/conduit.git
synced 2025-06-27 16:35:59 +00:00
feat(admin): list & query information about media
This commit is contained in:
parent
c3fb1b0456
commit
fd16e9c509
4 changed files with 772 additions and 3 deletions
|
@ -10,7 +10,10 @@ use crate::{
|
||||||
database::KeyValueDatabase,
|
database::KeyValueDatabase,
|
||||||
service::{
|
service::{
|
||||||
self,
|
self,
|
||||||
media::{BlockedMediaInfo, Data as _, DbFileMeta, MediaType},
|
media::{
|
||||||
|
BlockedMediaInfo, Data as _, DbFileMeta, FileInfo, MediaListItem, MediaQuery,
|
||||||
|
MediaQueryFileInfo, MediaQueryThumbInfo, MediaType, ServerNameOrUserId,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
services, utils, Error, Result,
|
services, utils, Error, Result,
|
||||||
};
|
};
|
||||||
|
@ -164,6 +167,117 @@ impl service::media::Data for KeyValueDatabase {
|
||||||
.ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Media not found."))
|
.ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Media not found."))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn query(&self, server_name: &ServerName, media_id: &str) -> Result<MediaQuery> {
|
||||||
|
let mut key = server_name.as_bytes().to_vec();
|
||||||
|
key.push(0xff);
|
||||||
|
key.extend_from_slice(media_id.as_bytes());
|
||||||
|
|
||||||
|
Ok(MediaQuery {
|
||||||
|
is_blocked: self.is_directly_blocked(server_name, media_id)?,
|
||||||
|
source_file: if let Some(DbFileMeta {
|
||||||
|
sha256_digest,
|
||||||
|
filename,
|
||||||
|
content_type,
|
||||||
|
unauthenticated_access_permitted,
|
||||||
|
}) = self
|
||||||
|
.servernamemediaid_metadata
|
||||||
|
.get(&key)?
|
||||||
|
.as_deref()
|
||||||
|
.map(parse_metadata)
|
||||||
|
.transpose()?
|
||||||
|
{
|
||||||
|
let sha256_hex = hex::encode(&sha256_digest);
|
||||||
|
|
||||||
|
let uploader_localpart = self
|
||||||
|
.servernamemediaid_userlocalpart
|
||||||
|
.get(&key)?
|
||||||
|
.as_deref()
|
||||||
|
.map(utils::string_from_bytes)
|
||||||
|
.transpose()
|
||||||
|
.map_err(|_| {
|
||||||
|
error!("Invalid UTF-8 for uploader of mxc://{server_name}/{media_id}");
|
||||||
|
Error::BadDatabase(
|
||||||
|
"Invalid UTF-8 in value of servernamemediaid_userlocalpart",
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let is_blocked_via_filehash = self.is_blocked_filehash(&sha256_digest)?;
|
||||||
|
|
||||||
|
let time_info = if let Some(filehash_meta) = self
|
||||||
|
.filehash_metadata
|
||||||
|
.get(&sha256_digest)?
|
||||||
|
.map(FilehashMetadata::from_vec)
|
||||||
|
{
|
||||||
|
Some(FileInfo {
|
||||||
|
creation: filehash_meta.creation(&sha256_digest)?,
|
||||||
|
last_access: filehash_meta.last_access(&sha256_digest)?,
|
||||||
|
size: filehash_meta.size(&sha256_digest)?,
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
Some(MediaQueryFileInfo {
|
||||||
|
uploader_localpart,
|
||||||
|
sha256_hex,
|
||||||
|
filename,
|
||||||
|
content_type,
|
||||||
|
unauthenticated_access_permitted,
|
||||||
|
is_blocked_via_filehash,
|
||||||
|
file_info: time_info,
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
},
|
||||||
|
thumbnails: {
|
||||||
|
key.push(0xff);
|
||||||
|
|
||||||
|
self.thumbnailid_metadata
|
||||||
|
.scan_prefix(key)
|
||||||
|
.map(|(k, v)| {
|
||||||
|
let (width, height) = dimensions_from_thumbnailid(&k)?;
|
||||||
|
|
||||||
|
let DbFileMeta {
|
||||||
|
sha256_digest,
|
||||||
|
filename,
|
||||||
|
content_type,
|
||||||
|
unauthenticated_access_permitted,
|
||||||
|
} = parse_metadata(&v)?;
|
||||||
|
|
||||||
|
let sha256_hex = hex::encode(&sha256_digest);
|
||||||
|
|
||||||
|
let is_blocked_via_filehash = self.is_blocked_filehash(&sha256_digest)?;
|
||||||
|
|
||||||
|
let time_info = if let Some(filehash_meta) = self
|
||||||
|
.filehash_metadata
|
||||||
|
.get(&sha256_digest)?
|
||||||
|
.map(FilehashMetadata::from_vec)
|
||||||
|
{
|
||||||
|
Some(FileInfo {
|
||||||
|
creation: filehash_meta.creation(&sha256_digest)?,
|
||||||
|
last_access: filehash_meta.last_access(&sha256_digest)?,
|
||||||
|
size: filehash_meta.size(&sha256_digest)?,
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(MediaQueryThumbInfo {
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
sha256_hex,
|
||||||
|
filename,
|
||||||
|
content_type,
|
||||||
|
unauthenticated_access_permitted,
|
||||||
|
is_blocked_via_filehash,
|
||||||
|
file_info: time_info,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect::<Result<_>>()?
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn purge_and_get_hashes(
|
fn purge_and_get_hashes(
|
||||||
&self,
|
&self,
|
||||||
media: &[(OwnedServerName, String)],
|
media: &[(OwnedServerName, String)],
|
||||||
|
@ -644,6 +758,336 @@ impl service::media::Data for KeyValueDatabase {
|
||||||
errors
|
errors
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn list(
|
||||||
|
&self,
|
||||||
|
server_name_or_user_id: Option<ServerNameOrUserId>,
|
||||||
|
include_thumbnails: bool,
|
||||||
|
content_type: Option<&str>,
|
||||||
|
before: Option<u64>,
|
||||||
|
after: Option<u64>,
|
||||||
|
) -> Result<Vec<MediaListItem>> {
|
||||||
|
let filter_medialistitem = |item: MediaListItem| {
|
||||||
|
if content_type.is_none_or(|ct_filter| {
|
||||||
|
item.content_type
|
||||||
|
.as_deref()
|
||||||
|
.map(|item_ct| {
|
||||||
|
if ct_filter.bytes().any(|char| char == b'/') {
|
||||||
|
item_ct == ct_filter
|
||||||
|
} else {
|
||||||
|
item_ct.starts_with(&(ct_filter.to_owned() + "/"))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.unwrap_or_default()
|
||||||
|
}) && before.is_none_or(|before| item.creation < before)
|
||||||
|
&& after.is_none_or(|after| item.creation > after)
|
||||||
|
{
|
||||||
|
Some(item)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let parse_servernamemediaid_metadata_iter =
|
||||||
|
|v: &[u8], next_part: Option<&[u8]>, server_name: &ServerName| {
|
||||||
|
let media_id_bytes = next_part.ok_or_else(|| {
|
||||||
|
Error::bad_database("Invalid format of key in servernamemediaid_metadata")
|
||||||
|
})?;
|
||||||
|
let media_id = utils::string_from_bytes(media_id_bytes).map_err(|_| {
|
||||||
|
Error::bad_database("Invalid Media ID String in servernamemediaid_metadata")
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let mut key = server_name.as_bytes().to_vec();
|
||||||
|
key.push(0xff);
|
||||||
|
key.extend_from_slice(media_id_bytes);
|
||||||
|
|
||||||
|
let uploader_localpart = self
|
||||||
|
.servernamemediaid_userlocalpart
|
||||||
|
.get(&key)?
|
||||||
|
.as_deref()
|
||||||
|
.map(utils::string_from_bytes)
|
||||||
|
.transpose()
|
||||||
|
.map_err(|_| {
|
||||||
|
error!("Invalid localpart of uploader for mxc://{server_name}/{media_id}");
|
||||||
|
Error::BadDatabase(
|
||||||
|
"Invalid uploader localpart in servernamemediaid_userlocalpart",
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let DbFileMeta {
|
||||||
|
sha256_digest,
|
||||||
|
filename,
|
||||||
|
content_type,
|
||||||
|
..
|
||||||
|
} = parse_metadata(v)?;
|
||||||
|
|
||||||
|
self.filehash_metadata
|
||||||
|
.get(&sha256_digest)?
|
||||||
|
.map(FilehashMetadata::from_vec)
|
||||||
|
.map(|meta| {
|
||||||
|
Ok(filter_medialistitem(MediaListItem {
|
||||||
|
server_name: server_name.to_owned(),
|
||||||
|
media_id: media_id.clone(),
|
||||||
|
uploader_localpart,
|
||||||
|
content_type,
|
||||||
|
filename,
|
||||||
|
dimensions: None,
|
||||||
|
size: meta.size(&sha256_digest)?,
|
||||||
|
creation: meta.creation(&sha256_digest)?,
|
||||||
|
}))
|
||||||
|
})
|
||||||
|
.transpose()
|
||||||
|
.map(Option::flatten)
|
||||||
|
};
|
||||||
|
|
||||||
|
let parse_thumbnailid_metadata_iter =
|
||||||
|
|k: &[u8], v: &[u8], media_id_part: Option<&[u8]>, server_name: &ServerName| {
|
||||||
|
let media_id_bytes = media_id_part.ok_or_else(|| {
|
||||||
|
Error::bad_database("Invalid format of key in servernamemediaid_metadata")
|
||||||
|
})?;
|
||||||
|
let media_id = utils::string_from_bytes(media_id_bytes).map_err(|_| {
|
||||||
|
Error::bad_database("Invalid Media ID String in servernamemediaid_metadata")
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let dimensions = dimensions_from_thumbnailid(k)?;
|
||||||
|
|
||||||
|
let DbFileMeta {
|
||||||
|
sha256_digest,
|
||||||
|
filename,
|
||||||
|
content_type,
|
||||||
|
..
|
||||||
|
} = parse_metadata(v)?;
|
||||||
|
|
||||||
|
self.filehash_metadata
|
||||||
|
.get(&sha256_digest)?
|
||||||
|
.map(FilehashMetadata::from_vec)
|
||||||
|
.map(|meta| {
|
||||||
|
Ok(filter_medialistitem(MediaListItem {
|
||||||
|
server_name: server_name.to_owned(),
|
||||||
|
media_id,
|
||||||
|
uploader_localpart: None,
|
||||||
|
content_type,
|
||||||
|
filename,
|
||||||
|
dimensions: Some(dimensions),
|
||||||
|
size: meta.size(&sha256_digest)?,
|
||||||
|
creation: meta.creation(&sha256_digest)?,
|
||||||
|
}))
|
||||||
|
})
|
||||||
|
.transpose()
|
||||||
|
.map(Option::flatten)
|
||||||
|
};
|
||||||
|
|
||||||
|
match server_name_or_user_id {
|
||||||
|
Some(ServerNameOrUserId::ServerName(server_name)) => {
|
||||||
|
let mut prefix = server_name.as_bytes().to_vec();
|
||||||
|
prefix.push(0xff);
|
||||||
|
|
||||||
|
let mut media = self
|
||||||
|
.servernamemediaid_metadata
|
||||||
|
.scan_prefix(prefix.clone())
|
||||||
|
.map(|(k, v)| {
|
||||||
|
let mut parts = k.rsplit(|b: &u8| *b == 0xff);
|
||||||
|
|
||||||
|
parse_servernamemediaid_metadata_iter(&v, parts.next(), &server_name)
|
||||||
|
})
|
||||||
|
.filter_map(Result::transpose)
|
||||||
|
.collect::<Result<Vec<_>>>()?;
|
||||||
|
|
||||||
|
if include_thumbnails {
|
||||||
|
media.append(
|
||||||
|
&mut self
|
||||||
|
.thumbnailid_metadata
|
||||||
|
.scan_prefix(prefix)
|
||||||
|
.map(|(k, v)| {
|
||||||
|
let mut parts = k.split(|b: &u8| *b == 0xff);
|
||||||
|
parts.next();
|
||||||
|
|
||||||
|
parse_thumbnailid_metadata_iter(&k, &v, parts.next(), &server_name)
|
||||||
|
})
|
||||||
|
.filter_map(Result::transpose)
|
||||||
|
.collect::<Result<Vec<_>>>()?,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(media)
|
||||||
|
}
|
||||||
|
Some(ServerNameOrUserId::UserId(user_id)) => {
|
||||||
|
let mut prefix = user_id.server_name().as_bytes().to_vec();
|
||||||
|
prefix.push(0xff);
|
||||||
|
prefix.extend_from_slice(user_id.localpart().as_bytes());
|
||||||
|
prefix.push(0xff);
|
||||||
|
|
||||||
|
self.servername_userlocalpart_mediaid
|
||||||
|
.scan_prefix(prefix)
|
||||||
|
.map(|(k, _)| -> Result<_> {
|
||||||
|
let mut parts = k.rsplit(|b: &u8| *b == 0xff);
|
||||||
|
|
||||||
|
let media_id_bytes = parts.next().ok_or_else(|| {
|
||||||
|
Error::bad_database(
|
||||||
|
"Invalid format of key in servername_userlocalpart_mediaid",
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let media_id = utils::string_from_bytes(media_id_bytes).map_err(|_| {
|
||||||
|
Error::bad_database(
|
||||||
|
"Invalid Media ID String in servername_userlocalpart_mediaid",
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let mut key = user_id.server_name().as_bytes().to_vec();
|
||||||
|
key.push(0xff);
|
||||||
|
key.extend_from_slice(media_id_bytes);
|
||||||
|
|
||||||
|
let Some(DbFileMeta {
|
||||||
|
sha256_digest,
|
||||||
|
filename,
|
||||||
|
content_type,
|
||||||
|
..
|
||||||
|
}) = self
|
||||||
|
.servernamemediaid_metadata
|
||||||
|
.get(&key)?
|
||||||
|
.as_deref()
|
||||||
|
.map(parse_metadata)
|
||||||
|
.transpose()?
|
||||||
|
else {
|
||||||
|
error!(
|
||||||
|
"Missing metadata for \"mxc://{}/{media_id}\", despite storing it's uploader",
|
||||||
|
user_id.server_name()
|
||||||
|
);
|
||||||
|
return Err(Error::BadDatabase(
|
||||||
|
"Missing metadata for media, despite storing it's uploader",
|
||||||
|
));
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut media = if let Some(item) = self
|
||||||
|
.filehash_metadata
|
||||||
|
.get(&sha256_digest)?
|
||||||
|
.map(FilehashMetadata::from_vec)
|
||||||
|
.map(|meta| {
|
||||||
|
Ok::<_, Error>(filter_medialistitem(MediaListItem {
|
||||||
|
server_name: user_id.server_name().to_owned(),
|
||||||
|
media_id: media_id.clone(),
|
||||||
|
uploader_localpart: Some(user_id.localpart().to_owned()),
|
||||||
|
content_type,
|
||||||
|
filename,
|
||||||
|
dimensions: None,
|
||||||
|
size: meta.size(&sha256_digest)?,
|
||||||
|
creation: meta.creation(&sha256_digest)?,
|
||||||
|
}))
|
||||||
|
})
|
||||||
|
.transpose()?
|
||||||
|
.flatten()
|
||||||
|
{
|
||||||
|
vec![item]
|
||||||
|
} else {
|
||||||
|
Vec::new()
|
||||||
|
};
|
||||||
|
|
||||||
|
if include_thumbnails {
|
||||||
|
key.push(0xff);
|
||||||
|
|
||||||
|
media.append(
|
||||||
|
&mut self
|
||||||
|
.thumbnailid_metadata
|
||||||
|
.scan_prefix(key)
|
||||||
|
.map(|(k, v)| {
|
||||||
|
let DbFileMeta {
|
||||||
|
sha256_digest,
|
||||||
|
filename,
|
||||||
|
content_type,
|
||||||
|
..
|
||||||
|
} = parse_metadata(&v)?;
|
||||||
|
|
||||||
|
let dimensions = dimensions_from_thumbnailid(&k)?;
|
||||||
|
|
||||||
|
self.filehash_metadata
|
||||||
|
.get(&sha256_digest)?
|
||||||
|
.map(FilehashMetadata::from_vec)
|
||||||
|
.map(|meta| {
|
||||||
|
Ok(filter_medialistitem(MediaListItem {
|
||||||
|
server_name: user_id.server_name().to_owned(),
|
||||||
|
media_id: media_id.clone(),
|
||||||
|
uploader_localpart: Some(
|
||||||
|
user_id.localpart().to_owned(),
|
||||||
|
),
|
||||||
|
content_type,
|
||||||
|
filename,
|
||||||
|
dimensions: Some(dimensions),
|
||||||
|
size: meta.size(&sha256_digest)?,
|
||||||
|
creation: meta.creation(&sha256_digest)?,
|
||||||
|
}))
|
||||||
|
})
|
||||||
|
.transpose()
|
||||||
|
.map(Option::flatten)
|
||||||
|
})
|
||||||
|
.filter_map(Result::transpose)
|
||||||
|
.collect::<Result<Vec<_>>>()?,
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(media)
|
||||||
|
})
|
||||||
|
.collect::<Result<Vec<Vec<_>>>>()
|
||||||
|
.map(|outer| outer.into_iter().flatten().collect::<Vec<MediaListItem>>())
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
let splitter = |b: &u8| *b == 0xff;
|
||||||
|
|
||||||
|
let get_servername = |parts: &mut Split<'_, u8, _>| -> Result<_> {
|
||||||
|
let server_name = parts
|
||||||
|
.next()
|
||||||
|
.ok_or_else(|| {
|
||||||
|
Error::bad_database(
|
||||||
|
"Invalid format of key in servernamemediaid_metadata",
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.map(utils::string_from_bytes)?
|
||||||
|
.map_err(|_| {
|
||||||
|
Error::bad_database(
|
||||||
|
"Invalid ServerName String in servernamemediaid_metadata",
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.map(OwnedServerName::try_from)?
|
||||||
|
.map_err(|_| {
|
||||||
|
Error::bad_database(
|
||||||
|
"Invalid ServerName String in servernamemediaid_metadata",
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
Ok(server_name)
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut media = self
|
||||||
|
.servernamemediaid_metadata
|
||||||
|
.iter()
|
||||||
|
.map(|(k, v)| {
|
||||||
|
let mut parts = k.split(splitter);
|
||||||
|
let server_name = get_servername(&mut parts)?;
|
||||||
|
|
||||||
|
parse_servernamemediaid_metadata_iter(&v, parts.next(), &server_name)
|
||||||
|
})
|
||||||
|
.filter_map(Result::transpose)
|
||||||
|
.collect::<Result<Vec<_>>>()?;
|
||||||
|
|
||||||
|
if include_thumbnails {
|
||||||
|
media.append(
|
||||||
|
&mut self
|
||||||
|
.thumbnailid_metadata
|
||||||
|
.iter()
|
||||||
|
.map(|(k, v)| {
|
||||||
|
let mut parts = k.split(splitter);
|
||||||
|
let server_name = get_servername(&mut parts)?;
|
||||||
|
|
||||||
|
parse_thumbnailid_metadata_iter(&k, &v, parts.next(), &server_name)
|
||||||
|
})
|
||||||
|
.filter_map(Result::transpose)
|
||||||
|
.collect::<Result<Vec<_>>>()?,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(media)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn list_blocked(&self) -> Vec<Result<BlockedMediaInfo>> {
|
fn list_blocked(&self) -> Vec<Result<BlockedMediaInfo>> {
|
||||||
let parse_servername = |parts: &mut Split<_, _>| {
|
let parse_servername = |parts: &mut Split<_, _>| {
|
||||||
OwnedServerName::try_from(
|
OwnedServerName::try_from(
|
||||||
|
@ -1216,6 +1660,21 @@ fn parse_metadata(value: &[u8]) -> Result<DbFileMeta> {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Attempts to parse the width and height from a "thumbnail id", returning the
|
||||||
|
/// width and height in that order
|
||||||
|
fn dimensions_from_thumbnailid(thumbnail_id: &[u8]) -> Result<(u32, u32)> {
|
||||||
|
let (width, height) = thumbnail_id[thumbnail_id
|
||||||
|
.len()
|
||||||
|
.checked_sub(8)
|
||||||
|
.ok_or_else(|| Error::BadDatabase("Invalid format of dimensions from thumbnailid"))?..]
|
||||||
|
.split_at(4);
|
||||||
|
|
||||||
|
Ok((
|
||||||
|
u32::from_be_bytes(width.try_into().expect("Length of slice is 4")),
|
||||||
|
u32::from_be_bytes(height.try_into().expect("Length of slice is 4")),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
pub struct FilehashMetadata {
|
pub struct FilehashMetadata {
|
||||||
value: Vec<u8>,
|
value: Vec<u8>,
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,6 +6,7 @@ use std::{
|
||||||
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
|
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
use bytesize::ByteSize;
|
||||||
use chrono::DateTime;
|
use chrono::DateTime;
|
||||||
use clap::{Args, Parser};
|
use clap::{Args, Parser};
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
|
@ -39,7 +40,13 @@ use crate::{
|
||||||
Error, PduEvent, Result,
|
Error, PduEvent, Result,
|
||||||
};
|
};
|
||||||
|
|
||||||
use super::{media::BlockedMediaInfo, pdu::PduBuilder};
|
use super::{
|
||||||
|
media::{
|
||||||
|
BlockedMediaInfo, FileInfo, MediaListItem, MediaQuery, MediaQueryFileInfo,
|
||||||
|
MediaQueryThumbInfo, ServerNameOrUserId,
|
||||||
|
},
|
||||||
|
pdu::PduBuilder,
|
||||||
|
};
|
||||||
|
|
||||||
#[cfg_attr(test, derive(Debug))]
|
#[cfg_attr(test, derive(Debug))]
|
||||||
#[derive(Parser)]
|
#[derive(Parser)]
|
||||||
|
@ -125,6 +132,52 @@ enum AdminCommand {
|
||||||
purge_media: DeactivatePurgeMediaArgs,
|
purge_media: DeactivatePurgeMediaArgs,
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/// Shows information about the requested media
|
||||||
|
QueryMedia {
|
||||||
|
/// The MXC URI of the media you want to request information about
|
||||||
|
mxc: Box<MxcUri>,
|
||||||
|
},
|
||||||
|
|
||||||
|
/// Lists all the media matching the specified requirements
|
||||||
|
ListMedia {
|
||||||
|
#[command(flatten)]
|
||||||
|
user_server_filter: ListMediaArgs,
|
||||||
|
|
||||||
|
/// Whether to include thumbnails in the list.
|
||||||
|
/// It is recommended to do so if you are not only looking
|
||||||
|
/// for local media, as with remote media, the full media file
|
||||||
|
/// might not be downloaded, just the thumbnail
|
||||||
|
#[arg(short = 't', long)]
|
||||||
|
include_thumbnails: bool,
|
||||||
|
|
||||||
|
#[arg(short, long)]
|
||||||
|
/// The content-type media must have to be listed.
|
||||||
|
/// if only a "type" (as opposed to "type/subtype") is specified,
|
||||||
|
/// all media with that type are returned, no matter the sub-type.
|
||||||
|
///
|
||||||
|
/// For example, if you request content-type "image", than files
|
||||||
|
/// of content type "image/png", "image/jpeg", etc. will be returned.
|
||||||
|
content_type: Option<String>,
|
||||||
|
|
||||||
|
#[arg(
|
||||||
|
short = 'b', long,
|
||||||
|
value_parser = humantime::parse_rfc3339_weak
|
||||||
|
)]
|
||||||
|
/// The point in time after which media had to be uploaded to be
|
||||||
|
/// shown (in the UTC timezone).
|
||||||
|
/// Should be in the format {YYYY}-{MM}-{DD}T{hh}:{mm}:{ss}
|
||||||
|
uploaded_before: Option<SystemTime>,
|
||||||
|
|
||||||
|
#[arg(
|
||||||
|
short = 'a', long,
|
||||||
|
value_parser = humantime::parse_rfc3339_weak
|
||||||
|
)]
|
||||||
|
/// The point in time before which media had to be uploaded to be
|
||||||
|
/// shown (in the UTC timezone).
|
||||||
|
/// Should be in the format {YYYY}-{MM}-{DD}T{hh}:{mm}:{ss}
|
||||||
|
uploaded_after: Option<SystemTime>,
|
||||||
|
},
|
||||||
|
|
||||||
/// Purge a list of media, formatted as MXC URIs
|
/// Purge a list of media, formatted as MXC URIs
|
||||||
/// There should be one URI per line, all contained within a code-block
|
/// There should be one URI per line, all contained within a code-block
|
||||||
///
|
///
|
||||||
|
@ -331,6 +384,22 @@ pub struct DeactivatePurgeMediaArgs {
|
||||||
force_filehash: bool,
|
force_filehash: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Args, Debug)]
|
||||||
|
#[group(required = false)]
|
||||||
|
pub struct ListMediaArgs {
|
||||||
|
#[arg(short, long)]
|
||||||
|
/// The user that uploaded the media.
|
||||||
|
/// Only local media uploaders can be recorded, so specifying a non-local
|
||||||
|
/// user will always yield no results
|
||||||
|
user: Option<Box<UserId>>,
|
||||||
|
|
||||||
|
#[arg(short, long)]
|
||||||
|
/// The server from which the media originated from.
|
||||||
|
/// If you want to list local media, just set this to
|
||||||
|
/// be your own server's servername
|
||||||
|
server: Option<Box<ServerName>>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub enum AdminRoomEvent {
|
pub enum AdminRoomEvent {
|
||||||
ProcessMessage(String),
|
ProcessMessage(String),
|
||||||
|
@ -960,6 +1029,162 @@ impl Service {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
AdminCommand::QueryMedia { mxc } => {
|
||||||
|
let Ok((server_name, media_id)) = mxc.parts() else {
|
||||||
|
return Ok(RoomMessageEventContent::text_plain("Invalid media MXC"));
|
||||||
|
};
|
||||||
|
|
||||||
|
let MediaQuery{ is_blocked, source_file, thumbnails } = services().media.query(server_name, media_id)?;
|
||||||
|
let mut message = format!("Is blocked Media ID: {is_blocked}");
|
||||||
|
|
||||||
|
if let Some(MediaQueryFileInfo {
|
||||||
|
uploader_localpart,
|
||||||
|
sha256_hex,
|
||||||
|
filename,
|
||||||
|
content_type,
|
||||||
|
unauthenticated_access_permitted,
|
||||||
|
is_blocked_via_filehash,
|
||||||
|
file_info: time_info,
|
||||||
|
}) = source_file {
|
||||||
|
message.push_str("\n\nInformation on full (non-thumbnail) file:\n");
|
||||||
|
|
||||||
|
if let Some(FileInfo {
|
||||||
|
creation,
|
||||||
|
last_access,
|
||||||
|
size,
|
||||||
|
}) = time_info {
|
||||||
|
message.push_str(&format!("\nIs stored: true\nCreated at: {}\nLast accessed at: {}\nSize of file: {}",
|
||||||
|
DateTime::from_timestamp(creation.try_into().unwrap_or(i64::MAX),0).expect("Timestamp is within range"),
|
||||||
|
DateTime::from_timestamp(last_access.try_into().unwrap_or(i64::MAX),0).expect("Timestamp is within range"),
|
||||||
|
ByteSize::b(size).display().si()
|
||||||
|
));
|
||||||
|
} else {
|
||||||
|
message.push_str("\nIs stored: false");
|
||||||
|
}
|
||||||
|
|
||||||
|
message.push_str(&format!("\nIs accessible via unauthenticated media endpoints: {unauthenticated_access_permitted}"));
|
||||||
|
message.push_str(&format!("\nSHA256 hash of file: {sha256_hex}"));
|
||||||
|
message.push_str(&format!("\nIs blocked due to sharing SHA256 hash with blocked media: {is_blocked_via_filehash}"));
|
||||||
|
|
||||||
|
if let Some(localpart) = uploader_localpart {
|
||||||
|
message.push_str(&format!("\nUploader: @{localpart}:{server_name}"))
|
||||||
|
}
|
||||||
|
if let Some(filename) = filename {
|
||||||
|
message.push_str(&format!("\nFilename: {filename}"))
|
||||||
|
}
|
||||||
|
if let Some(content_type) = content_type {
|
||||||
|
message.push_str(&format!("\nContent-type: {content_type}"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !thumbnails.is_empty() {
|
||||||
|
message.push_str("\n\nInformation on thumbnails of media:");
|
||||||
|
}
|
||||||
|
|
||||||
|
for MediaQueryThumbInfo{
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
sha256_hex,
|
||||||
|
filename,
|
||||||
|
content_type,
|
||||||
|
unauthenticated_access_permitted,
|
||||||
|
is_blocked_via_filehash,
|
||||||
|
file_info: time_info,
|
||||||
|
} in thumbnails {
|
||||||
|
message.push_str(&format!("\n\nDimensions: {width}x{height}"));
|
||||||
|
if let Some(FileInfo {
|
||||||
|
creation,
|
||||||
|
last_access,
|
||||||
|
size,
|
||||||
|
}) = time_info {
|
||||||
|
message.push_str(&format!("\nIs stored: true\nCreated at: {}\nLast accessed at: {}\nSize of file: {}",
|
||||||
|
DateTime::from_timestamp(creation.try_into().unwrap_or(i64::MAX),0).expect("Timestamp is within range"),
|
||||||
|
DateTime::from_timestamp(last_access.try_into().unwrap_or(i64::MAX),0).expect("Timestamp is within range"),
|
||||||
|
ByteSize::b(size).display().si()
|
||||||
|
));
|
||||||
|
} else {
|
||||||
|
message.push_str("\nIs stored: false");
|
||||||
|
}
|
||||||
|
|
||||||
|
message.push_str(&format!("\nIs accessible via unauthenticated media endpoints: {unauthenticated_access_permitted}"));
|
||||||
|
message.push_str(&format!("\nSHA256 hash of file: {sha256_hex}"));
|
||||||
|
message.push_str(&format!("\nIs blocked due to sharing SHA256 hash with blocked media: {is_blocked_via_filehash}"));
|
||||||
|
|
||||||
|
if let Some(filename) = filename {
|
||||||
|
message.push_str(&format!("\nFilename: {filename}"))
|
||||||
|
}
|
||||||
|
if let Some(content_type) = content_type {
|
||||||
|
message.push_str(&format!("\nContent-type: {content_type}"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
RoomMessageEventContent::text_plain(message)
|
||||||
|
}
|
||||||
|
AdminCommand::ListMedia {
|
||||||
|
user_server_filter: ListMediaArgs {
|
||||||
|
user,
|
||||||
|
server,
|
||||||
|
},
|
||||||
|
include_thumbnails,
|
||||||
|
content_type,
|
||||||
|
uploaded_before,
|
||||||
|
uploaded_after,
|
||||||
|
} => {
|
||||||
|
let mut markdown_message = String::from(
|
||||||
|
"| MXC URI | Dimensions (if thumbnail) | Created/Downloaded at | Uploader | Content-Type | Filename | Size |\n| --- | --- | --- | --- | --- | --- | --- |",
|
||||||
|
);
|
||||||
|
let mut html_message = String::from(
|
||||||
|
r#"<table><thead><tr><th scope="col">MXC URI</th><th scope="col">Dimensions (if thumbnail)</th><th scope="col">Created/Downloaded at</th><th scope="col">Uploader</th><th scope="col">Content-Type</th><th scope="col">Filename</th><th scope="col">Size</th></tr></thead><tbody>"#,
|
||||||
|
);
|
||||||
|
|
||||||
|
for MediaListItem{
|
||||||
|
server_name,
|
||||||
|
media_id,
|
||||||
|
uploader_localpart,
|
||||||
|
content_type,
|
||||||
|
filename,
|
||||||
|
dimensions,
|
||||||
|
size,
|
||||||
|
creation,
|
||||||
|
} in services().media.list(
|
||||||
|
user
|
||||||
|
.map(ServerNameOrUserId::UserId)
|
||||||
|
.or_else(|| server.map(ServerNameOrUserId::ServerName)),
|
||||||
|
include_thumbnails,
|
||||||
|
content_type.as_deref(),
|
||||||
|
uploaded_before
|
||||||
|
.map(|ts| ts.duration_since(UNIX_EPOCH))
|
||||||
|
.transpose()
|
||||||
|
.map_err(|_| Error::AdminCommand("Timestamp must be after unix epoch"))?
|
||||||
|
.as_ref()
|
||||||
|
.map(Duration::as_secs),
|
||||||
|
uploaded_after
|
||||||
|
.map(|ts| ts.duration_since(UNIX_EPOCH))
|
||||||
|
.transpose()
|
||||||
|
.map_err(|_| Error::AdminCommand("Timestamp must be after unix epoch"))?
|
||||||
|
.as_ref()
|
||||||
|
.map(Duration::as_secs)
|
||||||
|
)? {
|
||||||
|
|
||||||
|
let user_id = uploader_localpart.map(|localpart| format!("@{localpart}:{server_name}")).unwrap_or_default();
|
||||||
|
let content_type = content_type.unwrap_or_default();
|
||||||
|
let filename = filename.unwrap_or_default();
|
||||||
|
let dimensions = dimensions.map(|(w, h)| format!("{w}x{h}")).unwrap_or_default();
|
||||||
|
let size = ByteSize::b(size).display().si();
|
||||||
|
let creation = DateTime::from_timestamp(creation.try_into().unwrap_or(i64::MAX),0).expect("Timestamp is within range");
|
||||||
|
|
||||||
|
markdown_message
|
||||||
|
.push_str(&format!("\n| mxc://{server_name}/{media_id} | {dimensions} | {creation} | {user_id} | {content_type} | {filename} | {size} |"));
|
||||||
|
|
||||||
|
html_message.push_str(&format!(
|
||||||
|
"<tr><td>mxc://{server_name}/{media_id}</td><td>{dimensions}</td><td>{creation}</td><td>{user_id}</td><td>{content_type}</td><td>{filename}</td><td>{size}</td></tr>"
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
html_message.push_str("</tbody></table>");
|
||||||
|
|
||||||
|
RoomMessageEventContent::text_html(markdown_message, html_message)
|
||||||
|
},
|
||||||
AdminCommand::PurgeMedia => media_from_body(body).map_or_else(
|
AdminCommand::PurgeMedia => media_from_body(body).map_or_else(
|
||||||
|message| message,
|
|message| message,
|
||||||
|media| {
|
|media| {
|
||||||
|
|
|
@ -3,7 +3,9 @@ use sha2::{digest::Output, Sha256};
|
||||||
|
|
||||||
use crate::{config::MediaRetentionConfig, Error, Result};
|
use crate::{config::MediaRetentionConfig, Error, Result};
|
||||||
|
|
||||||
use super::{BlockedMediaInfo, DbFileMeta, MediaType};
|
use super::{
|
||||||
|
BlockedMediaInfo, DbFileMeta, MediaListItem, MediaQuery, MediaType, ServerNameOrUserId,
|
||||||
|
};
|
||||||
|
|
||||||
pub trait Data: Send + Sync {
|
pub trait Data: Send + Sync {
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
@ -44,6 +46,8 @@ pub trait Data: Send + Sync {
|
||||||
height: u32,
|
height: u32,
|
||||||
) -> Result<DbFileMeta>;
|
) -> Result<DbFileMeta>;
|
||||||
|
|
||||||
|
fn query(&self, server_name: &ServerName, media_id: &str) -> Result<MediaQuery>;
|
||||||
|
|
||||||
fn purge_and_get_hashes(
|
fn purge_and_get_hashes(
|
||||||
&self,
|
&self,
|
||||||
media: &[(OwnedServerName, String)],
|
media: &[(OwnedServerName, String)],
|
||||||
|
@ -83,6 +87,15 @@ pub trait Data: Send + Sync {
|
||||||
|
|
||||||
fn unblock(&self, media: &[(OwnedServerName, String)]) -> Vec<Error>;
|
fn unblock(&self, media: &[(OwnedServerName, String)]) -> Vec<Error>;
|
||||||
|
|
||||||
|
fn list(
|
||||||
|
&self,
|
||||||
|
server_name_or_user_id: Option<ServerNameOrUserId>,
|
||||||
|
include_thumbnails: bool,
|
||||||
|
content_type: Option<&str>,
|
||||||
|
before: Option<u64>,
|
||||||
|
after: Option<u64>,
|
||||||
|
) -> Result<Vec<MediaListItem>>;
|
||||||
|
|
||||||
/// Returns a Vec of:
|
/// Returns a Vec of:
|
||||||
/// - The server the media is from
|
/// - The server the media is from
|
||||||
/// - The media id
|
/// - The media id
|
||||||
|
|
|
@ -28,6 +28,55 @@ use tokio::{
|
||||||
io::{AsyncReadExt, AsyncWriteExt},
|
io::{AsyncReadExt, AsyncWriteExt},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
pub struct MediaQuery {
|
||||||
|
pub is_blocked: bool,
|
||||||
|
pub source_file: Option<MediaQueryFileInfo>,
|
||||||
|
pub thumbnails: Vec<MediaQueryThumbInfo>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct MediaQueryFileInfo {
|
||||||
|
pub uploader_localpart: Option<String>,
|
||||||
|
pub sha256_hex: String,
|
||||||
|
pub filename: Option<String>,
|
||||||
|
pub content_type: Option<String>,
|
||||||
|
pub unauthenticated_access_permitted: bool,
|
||||||
|
pub is_blocked_via_filehash: bool,
|
||||||
|
pub file_info: Option<FileInfo>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct MediaQueryThumbInfo {
|
||||||
|
pub width: u32,
|
||||||
|
pub height: u32,
|
||||||
|
pub sha256_hex: String,
|
||||||
|
pub filename: Option<String>,
|
||||||
|
pub content_type: Option<String>,
|
||||||
|
pub unauthenticated_access_permitted: bool,
|
||||||
|
pub is_blocked_via_filehash: bool,
|
||||||
|
pub file_info: Option<FileInfo>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct FileInfo {
|
||||||
|
pub creation: u64,
|
||||||
|
pub last_access: u64,
|
||||||
|
pub size: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct MediaListItem {
|
||||||
|
pub server_name: OwnedServerName,
|
||||||
|
pub media_id: String,
|
||||||
|
pub uploader_localpart: Option<String>,
|
||||||
|
pub content_type: Option<String>,
|
||||||
|
pub filename: Option<String>,
|
||||||
|
pub dimensions: Option<(u32, u32)>,
|
||||||
|
pub size: u64,
|
||||||
|
pub creation: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub enum ServerNameOrUserId {
|
||||||
|
ServerName(Box<ServerName>),
|
||||||
|
UserId(Box<UserId>),
|
||||||
|
}
|
||||||
|
|
||||||
pub struct FileMeta {
|
pub struct FileMeta {
|
||||||
pub content_disposition: ContentDisposition,
|
pub content_disposition: ContentDisposition,
|
||||||
pub content_type: Option<String>,
|
pub content_type: Option<String>,
|
||||||
|
@ -382,6 +431,11 @@ impl Service {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns information about the queried media
|
||||||
|
pub fn query(&self, server_name: &ServerName, media_id: &str) -> Result<MediaQuery> {
|
||||||
|
self.db.query(server_name, media_id)
|
||||||
|
}
|
||||||
|
|
||||||
/// Purges all of the specified media.
|
/// Purges all of the specified media.
|
||||||
///
|
///
|
||||||
/// If `force_filehash` is true, all media and/or thumbnails which share sha256 content hashes
|
/// If `force_filehash` is true, all media and/or thumbnails which share sha256 content hashes
|
||||||
|
@ -477,6 +531,24 @@ impl Service {
|
||||||
self.db.unblock(media)
|
self.db.unblock(media)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns a list of all the stored media, applying all the given filters to the results
|
||||||
|
pub fn list(
|
||||||
|
&self,
|
||||||
|
server_name_or_user_id: Option<ServerNameOrUserId>,
|
||||||
|
include_thumbnails: bool,
|
||||||
|
content_type: Option<&str>,
|
||||||
|
before: Option<u64>,
|
||||||
|
after: Option<u64>,
|
||||||
|
) -> Result<Vec<MediaListItem>> {
|
||||||
|
self.db.list(
|
||||||
|
server_name_or_user_id,
|
||||||
|
include_thumbnails,
|
||||||
|
content_type,
|
||||||
|
before,
|
||||||
|
after,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/// Returns a Vec of:
|
/// Returns a Vec of:
|
||||||
/// - The server the media is from
|
/// - The server the media is from
|
||||||
/// - The media id
|
/// - The media id
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue