ocre/storage/
resumable.rs

1//! Large files sent in parts, with resume: the requests and answers of
2//! [`multipart_uploads`](crate::storage::multipart_uploads), how a file is
3//! cut into parts, and the presigned `PUT` of each part.
4//!
5//! R2 multipart uploads take parts of the same size (at least 5 MiB, the
6//! last one smaller) numbered 1 to 10,000. A part the browser sends again
7//! replaces the previous one, so an interrupted upload resumes by sending
8//! the parts it did not finish.
9
10use std::collections::BTreeMap;
11
12use serde::{Deserialize, Serialize};
13
14use super::{Rules, S3Endpoint};
15use crate::{Error, Result, Validator};
16
17/// Size of the parts: 10 MiB, or more for a file that would need over 10,000.
18pub const PART_SIZE: u64 = 10 * MIB;
19
20/// Largest part sent through the Worker (its request limit is 100 MB).
21pub const MAX_WORKER_PART: u64 = 95 * MIB;
22
23/// Largest part R2 accepts: 5 GiB.
24pub const MAX_PART: u64 = 5 * 1024 * MIB;
25
26/// Most parts in one upload.
27pub const MAX_PARTS: u64 = 10_000;
28
29const MIB: u64 = 1024 * 1024;
30
31/// A started multipart upload, sent to the browser.
32///
33/// # Examples
34///
35/// ```
36/// let upload: ocre::storage::MultipartUpload =
37///     serde_json::from_str(r#"{"signed_key":"k.s","upload_id":"u","part_size":10485760,"part_count":3}"#).unwrap();
38/// assert_eq!(upload.part_count, 3);
39/// ```
40#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
41pub struct MultipartUpload {
42    /// The object key and its signature, as for a direct upload: the form submits it.
43    pub signed_key: String,
44    /// R2's id of the upload.
45    pub upload_id: String,
46    /// Bytes per part; the last part holds the rest.
47    pub part_size: u64,
48    /// Number of parts, numbered from 1.
49    pub part_count: u16,
50}
51
52/// The browser asks where to send parts: `{"signed_key", "upload_id", "parts": [1, 2, 3]}`.
53#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
54pub struct PartsRequest {
55    /// From the [`MultipartUpload`].
56    pub signed_key: String,
57    /// From the [`MultipartUpload`].
58    pub upload_id: String,
59    /// Part numbers, from 1; at most [`MAX_PARTS_PER_REQUEST`].
60    pub parts: Vec<u16>,
61}
62
63/// Most part URLs answered at once.
64pub const MAX_PARTS_PER_REQUEST: usize = 1_000;
65
66/// Where to `PUT` each part: `{"urls": {"1": "https://..."}}`.
67#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
68pub struct PartUrls {
69    /// Part number to URL: presigned on R2, or a route of the Worker.
70    pub urls: BTreeMap<u16, String>,
71}
72
73/// A part R2 stored: its number and the `ETag` it answered.
74#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
75pub struct CompletedPart {
76    /// From 1.
77    pub part_number: u16,
78    /// The part's `ETag`, quoted or not.
79    pub etag: String,
80}
81
82/// The browser finishes (`parts` set) or abandons an upload.
83#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
84pub struct FinishRequest {
85    /// From the [`MultipartUpload`].
86    pub signed_key: String,
87    /// From the [`MultipartUpload`].
88    pub upload_id: String,
89    /// Every part, as R2 answered them; empty to abort.
90    #[serde(default)]
91    pub parts: Vec<CompletedPart>,
92}
93
94/// How to cut `size` bytes into parts no larger than `max_part`: `(part_size, part_count)`.
95///
96/// # Errors
97///
98/// [`Error::Invalid`] on `field` when the file needs parts over `max_part`.
99///
100/// # Examples
101///
102/// ```
103/// use ocre::storage::{MAX_WORKER_PART, PART_SIZE, part_layout};
104///
105/// assert_eq!(part_layout("video", 25 * 1024 * 1024, MAX_WORKER_PART).unwrap(), (PART_SIZE, 3));
106/// assert_eq!(part_layout("video", 0, MAX_WORKER_PART).unwrap(), (PART_SIZE, 1));
107/// ```
108pub fn part_layout(field: &str, size: u64, max_part: u64) -> Result<(u64, u16)> {
109    // Over 10,000 parts of 10 MiB: larger parts, in whole MiB.
110    let part_size = PART_SIZE.max(size.div_ceil(MAX_PARTS).div_ceil(MIB) * MIB);
111    Validator::new()
112        .check(field, part_size > max_part, format!("is too large to upload in parts of at most {} MB", max_part / MIB))
113        .finish()?;
114    let count = size.div_ceil(part_size).max(1);
115    Ok((part_size, u16::try_from(count).expect("at most 10,000 parts")))
116}
117
118/// Checks the declared file against `rules` and lays it out in parts, as
119/// [`multipart_uploads`](crate::storage::multipart_uploads) does before
120/// creating the upload.
121///
122/// # Errors
123///
124/// [`Error::Invalid`] on `field` when the size or type breaks `rules`, or
125/// the file needs parts over `max_part`.
126pub fn check_multipart(field: &str, size: u64, content_type: &str, rules: &Rules, max_part: u64) -> Result<(u64, u16)> {
127    Validator::new().file_size_and_type(field, size, content_type, rules).finish()?;
128    part_layout(field, size, max_part)
129}
130
131/// Presigned `PUT` URLs of `parts` of the upload `upload_id` of `key`, valid `expires_in` seconds.
132///
133/// # Errors
134///
135/// [`Error::BadRequest`] for a part number outside 1 to 10,000 or too many
136/// parts at once; [`Error::Internal`] for an out-of-range `expires_in`.
137///
138/// # Examples
139///
140/// ```
141/// use ocre::storage::{S3Endpoint, presign_parts};
142///
143/// let r2 = S3Endpoint::r2("acc", "blog-storage", "AKID", "secret");
144/// let urls = presign_parts(&r2, "uploads/a", "up1", &[1, 2], 1_790_000_000, 3600).unwrap();
145/// assert!(urls.urls[&2].contains("partNumber=2&uploadId=up1"));
146/// ```
147pub fn presign_parts(
148    endpoint: &S3Endpoint,
149    key: &str,
150    upload_id: &str,
151    parts: &[u16],
152    now: i64,
153    expires_in: u64,
154) -> Result<PartUrls> {
155    check_part_numbers(parts)?;
156    let mut urls = BTreeMap::new();
157    for &part in parts {
158        let number = part.to_string();
159        let query = [("partNumber", number.as_str()), ("uploadId", upload_id)];
160        urls.insert(part, endpoint.presign("PUT", key, &[], &query, now, expires_in)?);
161    }
162    Ok(PartUrls { urls })
163}
164
165/// Part numbers are 1 to 10,000, at most [`MAX_PARTS_PER_REQUEST`] at once.
166pub(crate) fn check_part_numbers(parts: &[u16]) -> Result<()> {
167    if parts.len() > MAX_PARTS_PER_REQUEST {
168        return Err(Error::bad_request(format!("ask for at most {MAX_PARTS_PER_REQUEST} parts at once")));
169    }
170    if let Some(part) = parts.iter().find(|&&part| part == 0 || u64::from(part) > MAX_PARTS) {
171        return Err(Error::bad_request(format!("part {part} is not between 1 and {MAX_PARTS}")));
172    }
173    Ok(())
174}
175
176#[cfg(test)]
177#[path = "../../tests/storage/resumable.rs"]
178mod tests;
ocre/storage/resumable.rs

ocre/storage/
resumable.rs
