oss/conduit
oss/conduit
1
0
Fork 0
mirror of https://gitlab.com/famedly/conduit.git synced 2025-06-27 16:35:59 +00:00
Code Wiki Activity

feat(service/media): add deep hashed directory structure for S3

Browse source
This commit is contained in:
AndSDev 2025-06-06 17:19:18 +03:00
parent 2c47045659
commit 59e4f4d8f1
3 changed files with 64 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

13
src/config/mod.rs
View file

@ -250,6 +250,8 @@ impl From<IncompleteConfig> for Config {
secret,
duration,
bucket_use_path,
path,
directory_structure,
} => {
let path_style = if bucket_use_path {
rusty_s3::UrlStyle::Path
@ -263,9 +265,11 @@ impl From<IncompleteConfig> for Config {
let credentials = rusty_s3::Credentials::new(key, secret);
MediaBackendConfig::S3 {
bucket: bucket,
credentials: credentials,
bucket,
credentials,
duration: Duration::from_secs(duration),
path,
directory_structure,
}
}
},
@ -517,6 +521,9 @@ pub enum IncompleteMediaBackendConfig {
duration: u64,
#[serde(default = "false_fn")]
bucket_use_path: bool,
path: Option<String>,
#[serde(default)]
directory_structure: DirectoryStructure,
},
}
@ -539,6 +546,8 @@ pub enum MediaBackendConfig {
bucket: rusty_s3::Bucket,
credentials: rusty_s3::Credentials,
duration: Duration,
path: Option<String>,
directory_structure: DirectoryStructure,
},
}

45
src/service/globals/mod.rs
View file

@ -488,23 +488,40 @@ impl Service {
directory_structure: &DirectoryStructure,
sha256_hex: &str,
) -> Result<PathBuf> {
let mut r = PathBuf::new();
r.push(media_directory);
Ok(PathBuf::from_iter(self.split_media_path(
Some(media_directory),
directory_structure,
sha256_hex,
)?))
}
if let DirectoryStructure::Deep { length, depth } = directory_structure {
let mut filename = sha256_hex;
for _ in 0..depth.get() {
let (current_path, next) = filename.split_at(length.get().into());
filename = next;
r.push(current_path);
pub fn split_media_path<'a>(
&self,
media_directory: Option<&'a str>,
directory_structure: &DirectoryStructure,
sha256_hex: &'a str,
) -> Result<Vec<&'a str>> {
match directory_structure {
DirectoryStructure::Flat => match media_directory {
Some(path) => Ok(vec![path, sha256_hex]),
None => Ok(vec![sha256_hex]),
},
DirectoryStructure::Deep { length, depth } => {
let mut r: Vec<&'a str> = Vec::with_capacity((depth.get() + 2).into());
if let Some(path) = media_directory {
r.push(path);
}
let mut filename = sha256_hex;
for _ in 0..depth.get() {
let (current_path, next) = filename.split_at(length.get().into());
filename = next;
r.push(current_path);
}
r.push(filename);
Ok(r)
}
r.push(filename);
} else {
r.push(sha256_hex);
}
Ok(r)
}
pub fn shutdown(&self) {

25
src/service/media/mod.rs
View file

@ -621,10 +621,16 @@ impl Service {
bucket,
credentials,
duration,
path,
directory_structure,
} => {
let sha256_hex = hex::encode(sha256_digest);
let file_name = services()
.globals
.split_media_path(path.as_deref(), directory_structure, &sha256_hex)?
.join("/");
let url = bucket
.get_object(Some(credentials), &sha256_hex)
.get_object(Some(credentials), &file_name)
.sign(*duration);
let client = services().globals.default_client();
@ -680,9 +686,16 @@ pub async fn create_file(sha256_hex: &str, file: &[u8]) -> Result<()> {
bucket,
credentials,
duration,
path,
directory_structure,
} => {
let file_name = services()
.globals
.split_media_path(path.as_deref(), directory_structure, sha256_hex)?
.join("/");
let url = bucket
.put_object(Some(credentials), sha256_hex)
.put_object(Some(credentials), &file_name)
.sign(*duration);
let client = services().globals.default_client();
@ -767,9 +780,15 @@ async fn delete_file(sha256_hex: &str) -> Result<()> {
bucket,
credentials,
duration,
path,
directory_structure,
} => {
let file_name = services()
.globals
.split_media_path(path.as_deref(), directory_structure, sha256_hex)?
.join("/");
let url = bucket
.delete_object(Some(credentials), sha256_hex)
.delete_object(Some(credentials), &file_name)
.sign(*duration);
let client = services().globals.default_client();