Skip to main content

objectstore_server/extractors/
batch.rs

1//! Axum extractor for batch operation streams.
2//!
3//! [`BatchOperationStream`] reads a multipart request body one part at a time, detects the
4//! [`BatchFormat`] from the first part, and maps the parts onto a lazy stream of [`Operation`]s.
5//!
6//! All insert metadata uses the shared request timestamp, even for fields parsed
7//! later while streaming the body.
8//!
9//! An invalid operation fails on its own, and reading continues with the next part. A broken
10//! multipart body, such as a truncated body or unparseable part headers, yields one error and
11//! ends the stream, because no further part can be found after it.
12//!
13//! # JSON layout
14//!
15//! Each operation is a descriptor part named by its index (`"0"`, `"1"`, …), whose body is a
16//! JSON object with `kind`, `key` (optional for inserts) and, for inserts, an optional
17//! [`WriteMetadata`] object. An insert is directly followed by its payload part
18//! `"<i>.payload"`. The parser applies these rules:
19//!
20//! - A part whose name ends in `.payload` is a payload part. Every other part is a descriptor.
21//! - The index of an operation is the position of its descriptor among descriptor parts. A
22//!   descriptor with another name, invalid JSON, or a missing required field fails with `400`.
23//! - An insert without its payload part fails with `400`, and the next part is read as the next
24//!   descriptor. Any other payload part is skipped.
25//! - The stored content type is `metadata.content_type`. If that is absent and the payload is
26//!   not encoded, it is the payload part's `Content-Type`, and otherwise
27//!   `application/octet-stream`.
28//!
29//! Both layouts allow 1000 operations per request and 1 MiB per part. The JSON layout counts
30//! descriptor parts toward the operation limit and caps each descriptor at 64 KiB.
31
32use std::fmt::Debug;
33use std::pin::Pin;
34
35use axum::RequestExt;
36use axum::extract::{
37    FromRequest, Multipart, Request,
38    multipart::{Field, MultipartError, MultipartRejection},
39};
40use bytes::{Bytes, BytesMut};
41use futures::stream::{self, BoxStream, Peekable};
42use futures::{Stream, StreamExt};
43use http::header::CONTENT_TYPE;
44use http::{HeaderMap, StatusCode};
45use objectstore_service::id::ObjectKey;
46use objectstore_service::streaming::{Delete, Get, Head, Insert, Operation};
47use objectstore_types::batch::{
48    self, Descriptor, MAX_BATCH_OPERATIONS, MAX_BATCH_PART_SIZE, MAX_DESCRIPTOR_SIZE,
49    OperationKind, PAYLOAD_SUFFIX,
50};
51use objectstore_types::headers;
52use objectstore_types::metadata::{Metadata, WriteMetadata};
53use objectstore_types::time::Timestamp;
54use thiserror::Error;
55
56use crate::batch::{HEADER_BATCH_OPERATION_KEY, HEADER_BATCH_OPERATION_KIND};
57use crate::extractors::request_time::RequestTime;
58use crate::rate_limits::LimitExceeded;
59
60/// Errors that can occur when processing or executing batch operations.
61#[derive(Debug, Error)]
62pub enum BatchError {
63    /// Malformed request.
64    #[error("bad request: {0}")]
65    BadRequest(String),
66
67    /// Errors in parsing or reading a multipart request body.
68    #[error("multipart error: {0}")]
69    Multipart(#[from] MultipartError),
70
71    /// An operation descriptor of the JSON layout that cannot be deserialized.
72    #[error("invalid operation descriptor: {0}")]
73    Descriptor(#[from] serde_json::Error),
74
75    /// Errors related to de/serialization and parsing of object metadata.
76    #[error("metadata error: {0}")]
77    Metadata(#[from] objectstore_types::metadata::Error),
78
79    /// Size or cardinality limit exceeded.
80    #[error("batch limit exceeded: {0}")]
81    LimitExceeded(String),
82
83    /// Operation rejected due to rate limiting.
84    #[error("{0}")]
85    RateLimited(LimitExceeded),
86
87    /// Errors encountered when serializing batch response parts.
88    #[error("response part serialization error: {context}")]
89    ResponseSerialization {
90        /// Context describing what was being serialized.
91        context: String,
92        /// The underlying error.
93        #[source]
94        cause: Box<dyn std::error::Error + Send + Sync>,
95    },
96}
97
98impl BatchError {
99    /// Returns the HTTP status code reported for the failed operation.
100    pub fn status(&self) -> StatusCode {
101        match self {
102            Self::BadRequest(_) | Self::Multipart(_) | Self::Descriptor(_) | Self::Metadata(_) => {
103                StatusCode::BAD_REQUEST
104            }
105            Self::LimitExceeded(_) => StatusCode::PAYLOAD_TOO_LARGE,
106            Self::RateLimited(rejection) => rejection.status(),
107            Self::ResponseSerialization { .. } => StatusCode::INTERNAL_SERVER_ERROR,
108        }
109    }
110
111    fn too_many_operations() -> Self {
112        Self::LimitExceeded(format!(
113            "exceeded {MAX_BATCH_OPERATIONS} operations per batch request"
114        ))
115    }
116}
117
118/// The part layout of a batch request. The response uses the same layout.
119#[derive(Clone, Copy, Debug, PartialEq, Eq)]
120pub enum BatchFormat {
121    /// One part per operation, described by `x-sn-batch-operation-*` and metadata part headers.
122    ///
123    /// Detected when the first part has an
124    /// [`x-sn-batch-operation-kind`](HEADER_BATCH_OPERATION_KIND) header.
125    Headers,
126    /// JSON descriptor parts plus payload parts, as described in the [module docs](self).
127    ///
128    /// Used for every other request, including an empty one.
129    Json,
130}
131
132/// A multipart part, read into memory.
133#[derive(Debug)]
134struct Part {
135    name: Option<String>,
136    headers: HeaderMap,
137    /// The body, or [`BatchError::LimitExceeded`] if it is larger than 1 MiB.
138    body: Result<Bytes, BatchError>,
139}
140
141impl Part {
142    /// Returns whether this is a payload part, named `"<i>.payload"`.
143    fn is_payload(&self) -> bool {
144        self.name
145            .as_deref()
146            .is_some_and(|name| name.ends_with(PAYLOAD_SUFFIX))
147    }
148
149    /// Returns whether this part describes an operation of the header layout.
150    fn has_kind_header(&self) -> bool {
151        self.headers.contains_key(HEADER_BATCH_OPERATION_KIND)
152    }
153
154    /// Returns the part's `Content-Type`, if it is a valid string.
155    fn content_type(&self) -> Option<String> {
156        self.headers
157            .get(CONTENT_TYPE)?
158            .to_str()
159            .ok()
160            .map(str::to_owned)
161    }
162
163    /// Parses this part as the descriptor of the operation at `index`.
164    fn into_descriptor(self, index: usize) -> Result<Descriptor, BatchError> {
165        if index >= MAX_BATCH_OPERATIONS {
166            return Err(BatchError::too_many_operations());
167        }
168
169        if self.name.as_deref().and_then(|name| name.parse().ok()) != Some(index) {
170            return Err(BatchError::BadRequest(format!(
171                "expected descriptor part named \"{index}\", found {:?}",
172                self.name
173            )));
174        }
175
176        let body = self.body?;
177        if body.len() > MAX_DESCRIPTOR_SIZE {
178            return Err(BatchError::LimitExceeded(format!(
179                "operation descriptor exceeds size limit of {MAX_DESCRIPTOR_SIZE} bytes"
180            )));
181        }
182
183        Ok(serde_json::from_slice(&body)?)
184    }
185}
186
187/// The parts of a request, read one at a time. Ends after the first broken part.
188type Parts = Pin<Box<Peekable<BoxStream<'static, Result<Part, BatchError>>>>>;
189
190/// Reads the body of a field, failing once it exceeds `MAX_BATCH_PART_SIZE`.
191async fn read_body(mut field: Field<'_>) -> Result<Bytes, BatchError> {
192    let mut body = BytesMut::new();
193    while let Some(chunk) = field.chunk().await? {
194        if body.len() + chunk.len() > MAX_BATCH_PART_SIZE {
195            return Err(BatchError::LimitExceeded(format!(
196                "individual request in batch exceeds body size limit of {MAX_BATCH_PART_SIZE} bytes"
197            )));
198        }
199        body.extend_from_slice(&chunk);
200    }
201    Ok(body.freeze())
202}
203
204/// Streams the parts of a multipart body, each read into memory.
205fn stream_parts(multipart: Multipart) -> Parts {
206    let parts = stream::unfold(Some(multipart), |multipart| async move {
207        let mut multipart = multipart?;
208        let field = match multipart.next_field().await {
209            Ok(Some(field)) => field,
210            Ok(None) => return None,
211            Err(error) => return Some((Err(error.into()), None)),
212        };
213
214        let name = field.name().map(str::to_owned);
215        let headers = field.headers().clone();
216        match read_body(field).await {
217            Err(error @ BatchError::Multipart(_)) => Some((Err(error), None)),
218            body => Some((
219                Ok(Part {
220                    name,
221                    headers,
222                    body,
223                }),
224                Some(multipart),
225            )),
226        }
227    });
228
229    Box::pin(parts.boxed().peekable())
230}
231
232/// Parses a part of the header layout into an operation.
233fn parse_header_part(part: Part, access_time: Timestamp) -> Result<Operation, BatchError> {
234    let kind_name = part
235        .headers
236        .get(HEADER_BATCH_OPERATION_KIND)
237        .ok_or_else(|| {
238            BatchError::BadRequest(format!("missing {HEADER_BATCH_OPERATION_KIND} header"))
239        })?;
240    let kind_name = kind_name
241        .to_str()
242        .map_err(|_| {
243            BatchError::BadRequest(format!(
244                "unable to convert {HEADER_BATCH_OPERATION_KIND} header value to string"
245            ))
246        })?
247        .to_lowercase();
248    let kind = OperationKind::from(kind_name.as_str());
249
250    let key = part
251        .headers
252        .get(HEADER_BATCH_OPERATION_KEY)
253        .map(|v| {
254            headers::decode_header_value(v).map_err(|_| {
255                BatchError::BadRequest(format!(
256                    "unable to percent-decode {HEADER_BATCH_OPERATION_KEY} header value"
257                ))
258            })
259        })
260        .transpose()?;
261    let required_key = |key: Option<ObjectKey>| {
262        key.ok_or_else(|| {
263            BatchError::BadRequest(format!(
264                "missing {HEADER_BATCH_OPERATION_KEY} header for {kind_name} operation"
265            ))
266        })
267    };
268
269    let operation = match kind {
270        OperationKind::Get => Operation::Get(Get {
271            key: required_key(key)?,
272        }),
273        OperationKind::Delete => Operation::Delete(Delete {
274            key: required_key(key)?,
275        }),
276        OperationKind::Head => Operation::Head(Head {
277            key: required_key(key)?,
278        }),
279        OperationKind::Insert => Operation::Insert(Box::new(Insert {
280            metadata: Metadata::from_insert_headers(&part.headers, "", access_time)?,
281            payload: part.body?,
282            key,
283        })),
284        OperationKind::Unknown => {
285            return Err(BatchError::BadRequest(format!(
286                "invalid operation kind: {kind_name}"
287            )));
288        }
289    };
290    Ok(operation)
291}
292
293/// Streams the operations of a header-layout request.
294fn stream_header_ops(
295    parts: Parts,
296    access_time: Timestamp,
297) -> impl Stream<Item = Result<Operation, BatchError>> {
298    parts.enumerate().map(move |(index, part)| {
299        if index >= MAX_BATCH_OPERATIONS {
300            return Err(BatchError::too_many_operations());
301        }
302        parse_header_part(part?, access_time)
303    })
304}
305
306/// Builds an insert from its descriptor and the part that follows it, if that is its payload.
307fn parse_insert(
308    key: Option<ObjectKey>,
309    mut metadata: WriteMetadata,
310    payload: Option<Result<Part, BatchError>>,
311    access_time: Timestamp,
312) -> Result<Operation, BatchError> {
313    let payload = payload.ok_or_else(|| {
314        BatchError::BadRequest("missing payload part for insert operation".to_owned())
315    })??;
316
317    if metadata.content_type.is_none() && metadata.content_encoding.is_none() {
318        metadata.content_type = payload.content_type();
319    }
320
321    Ok(Operation::Insert(Box::new(Insert {
322        key,
323        metadata: metadata.into_metadata(access_time)?,
324        payload: payload.body?,
325    })))
326}
327
328/// Streams the operations of a JSON-layout request.
329fn stream_json_ops(
330    parts: Parts,
331    access_time: Timestamp,
332) -> impl Stream<Item = Result<Operation, BatchError>> {
333    stream::unfold((parts, 0), move |(mut parts, index)| async move {
334        // Payload parts that do not directly follow their insert are silently skipped.
335        let part = loop {
336            match parts.next().await? {
337                Ok(part) if part.is_payload() => continue,
338                part => break part,
339            }
340        };
341
342        let operation = match part.and_then(|part| part.into_descriptor(index)) {
343            Ok(Descriptor::Get { key }) => Ok(Operation::Get(Get { key })),
344            Ok(Descriptor::Head { key }) => Ok(Operation::Head(Head { key })),
345            Ok(Descriptor::Delete { key }) => Ok(Operation::Delete(Delete { key })),
346            Ok(Descriptor::Insert { key, metadata }) => {
347                // The next part is the payload, unless it belongs elsewhere. A broken body fails
348                // the insert.
349                let payload_name = batch::payload_part_name(index);
350                let is_payload = |next: &Result<Part, BatchError>| match next {
351                    Ok(part) => part.name.as_deref() == Some(payload_name.as_str()),
352                    Err(_) => true,
353                };
354                let payload = parts.as_mut().next_if(is_payload).await;
355                parse_insert(key, metadata, payload, access_time)
356            }
357            Err(error) => Err(error),
358        };
359
360        Some((operation, (parts, index + 1)))
361    })
362}
363
364/// A lazily-parsed stream of batch operations extracted from a multipart request body.
365///
366/// The stream yields exactly one item per operation in index order, so the position of an item
367/// is the index of its operation. An empty body yields an empty stream.
368pub struct BatchOperationStream {
369    /// The layout of the request, which the response uses as well.
370    pub format: BatchFormat,
371    /// The operations in index order.
372    pub operations: BoxStream<'static, Result<Operation, BatchError>>,
373}
374
375impl Debug for BatchOperationStream {
376    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
377        f.debug_struct("BatchOperationStream")
378            .field("format", &self.format)
379            .finish_non_exhaustive()
380    }
381}
382
383impl<S> FromRequest<S> for BatchOperationStream
384where
385    S: Send + Sync,
386{
387    type Rejection = MultipartRejection;
388
389    async fn from_request(mut request: Request, state: &S) -> Result<Self, Self::Rejection> {
390        let Ok(RequestTime(access_time)) = request.extract_parts::<RequestTime>().await;
391
392        let mut parts = stream_parts(Multipart::from_request(request, state).await?);
393        let format = match parts.as_mut().peek().await {
394            Some(Ok(part)) if part.has_kind_header() => BatchFormat::Headers,
395            _ => BatchFormat::Json,
396        };
397
398        let operations = match format {
399            BatchFormat::Headers => stream_header_ops(parts, access_time).boxed(),
400            BatchFormat::Json => stream_json_ops(parts, access_time).boxed(),
401        };
402
403        Ok(Self { format, operations })
404    }
405}
406
407#[cfg(test)]
408mod tests {
409    use std::time::Duration;
410
411    use super::*;
412    use axum::body::Body;
413    use axum::http::{Request, header::CONTENT_TYPE};
414    use futures::StreamExt;
415    use objectstore_service::streaming::Operation;
416    use objectstore_types::headers;
417    use objectstore_types::metadata::{ExpirationPolicy, HEADER_EXPIRATION, HEADER_ORIGIN};
418
419    #[tokio::test]
420    async fn test_valid_request_works() {
421        let insert1_data = b"first blob data";
422        let insert2_data = b"second blob data";
423        let expiration = ExpirationPolicy::TimeToLive(Duration::from_hours(1));
424        let body = format!(
425            "--boundary\r\n\
426             {HEADER_BATCH_OPERATION_KEY}: {key0}\r\n\
427             {HEADER_BATCH_OPERATION_KIND}: get\r\n\
428             \r\n\
429             \r\n\
430             --boundary\r\n\
431             {HEADER_BATCH_OPERATION_KEY}: {key1}\r\n\
432             {HEADER_BATCH_OPERATION_KIND}: insert\r\n\
433             Content-Type: application/octet-stream\r\n\
434             \r\n\
435             {insert1}\r\n\
436             --boundary\r\n\
437             {HEADER_BATCH_OPERATION_KEY}: {key2}\r\n\
438             {HEADER_BATCH_OPERATION_KIND}: insert\r\n\
439             {HEADER_EXPIRATION}: {expiration}\r\n\
440             {HEADER_ORIGIN}: 203.0.113.42\r\n\
441             Content-Type: text/plain\r\n\
442             \r\n\
443             {insert2}\r\n\
444             --boundary\r\n\
445             {HEADER_BATCH_OPERATION_KEY}: {key3}\r\n\
446             {HEADER_BATCH_OPERATION_KIND}: delete\r\n\
447             \r\n\
448             \r\n\
449             --boundary--\r\n",
450            key0 = "test%2F0", // "test/0" percent-encoded
451            key1 = "test1",
452            key2 = "test2",
453            key3 = "test3",
454            insert1 = String::from_utf8_lossy(insert1_data),
455            insert2 = String::from_utf8_lossy(insert2_data),
456        );
457
458        let access_time = Timestamp::UNIX_EPOCH;
459        let request = Request::builder()
460            .extension(RequestTime(access_time))
461            .header(CONTENT_TYPE, "multipart/form-data; boundary=boundary")
462            .body(Body::from(body))
463            .unwrap();
464
465        let batch_request = BatchOperationStream::from_request(request, &())
466            .await
467            .unwrap();
468
469        let operations: Vec<_> = batch_request.operations.collect().await;
470        assert_eq!(operations.len(), 4);
471
472        let Operation::Get(get_op) = &operations[0].as_ref().unwrap() else {
473            panic!("expected get operation");
474        };
475        assert_eq!(get_op.key, "test/0");
476
477        let Operation::Insert(insert_op1) = &operations[1].as_ref().unwrap() else {
478            panic!("expected insert operation");
479        };
480        assert_eq!(insert_op1.key.as_deref(), Some("test1"));
481        assert_eq!(insert_op1.metadata.content_type, "application/octet-stream");
482        assert_eq!(insert_op1.metadata.origin, None);
483        assert_eq!(insert_op1.metadata.time_created, Some(access_time));
484        assert_eq!(insert_op1.payload.as_ref(), insert1_data);
485
486        let Operation::Insert(insert_op2) = &operations[2].as_ref().unwrap() else {
487            panic!("expected insert operation");
488        };
489        assert_eq!(insert_op2.key.as_deref(), Some("test2"));
490        assert_eq!(insert_op2.metadata.content_type, "text/plain");
491        assert_eq!(insert_op2.metadata.expiration_policy, expiration);
492        assert_eq!(insert_op2.metadata.time_created, Some(access_time));
493        assert_eq!(
494            insert_op2.metadata.time_expires,
495            Some(access_time + Duration::from_hours(1))
496        );
497        assert_eq!(insert_op2.metadata.origin.as_deref(), Some("203.0.113.42"));
498        assert_eq!(insert_op2.payload.as_ref(), insert2_data);
499
500        let Operation::Delete(delete_op) = &operations[3].as_ref().unwrap() else {
501            panic!("expected delete operation");
502        };
503        assert_eq!(delete_op.key, "test3");
504    }
505
506    #[tokio::test]
507    async fn test_insert_without_key_header() {
508        let body = format!(
509            "--boundary\r\n\
510             {HEADER_BATCH_OPERATION_KIND}: insert\r\n\
511             Content-Type: application/octet-stream\r\n\
512             \r\n\
513             keyless payload\r\n\
514             --boundary--\r\n",
515        );
516
517        let request = Request::builder()
518            .header(CONTENT_TYPE, "multipart/form-data; boundary=boundary")
519            .body(Body::from(body))
520            .unwrap();
521
522        let batch_request = BatchOperationStream::from_request(request, &())
523            .await
524            .unwrap();
525
526        let operations: Vec<_> = batch_request.operations.collect().await;
527        assert_eq!(operations.len(), 1);
528
529        let Operation::Insert(insert_op) = &operations[0].as_ref().unwrap() else {
530            panic!("expected insert operation");
531        };
532        assert!(insert_op.key.is_none());
533        assert_eq!(insert_op.payload.as_ref(), b"keyless payload");
534    }
535
536    #[tokio::test]
537    async fn test_individual_errors_with_isolation() {
538        let large_payload = "x".repeat(MAX_BATCH_PART_SIZE + 1);
539        let valid_key = headers::encode_header_str("valid");
540        let body = format!(
541            "--boundary\r\n\
542             {HEADER_BATCH_OPERATION_KIND}: get\r\n\
543             \r\n\
544             \r\n\
545             --boundary\r\n\
546             {HEADER_BATCH_OPERATION_KEY}: {valid_key}\r\n\
547             {HEADER_BATCH_OPERATION_KIND}: get\r\n\
548             \r\n\
549             \r\n\
550             --boundary\r\n\
551             {HEADER_BATCH_OPERATION_KIND}: delete\r\n\
552             \r\n\
553             \r\n\
554             --boundary\r\n\
555             {HEADER_BATCH_OPERATION_KEY}: {valid_key}\r\n\
556             {HEADER_BATCH_OPERATION_KIND}: insert\r\n\
557             Content-Type: application/octet-stream\r\n\
558             \r\n\
559             {large_payload}\r\n\
560             --boundary\r\n\
561             {HEADER_BATCH_OPERATION_KEY}: {valid_key}\r\n\
562             {HEADER_BATCH_OPERATION_KIND}: delete\r\n\
563             \r\n\
564             \r\n\
565             --boundary--\r\n",
566        );
567
568        let request = Request::builder()
569            .header(CONTENT_TYPE, "multipart/form-data; boundary=boundary")
570            .body(Body::from(body))
571            .unwrap();
572
573        let batch_request = BatchOperationStream::from_request(request, &())
574            .await
575            .unwrap();
576
577        let operations: Vec<_> = batch_request.operations.collect().await;
578        assert_eq!(operations.len(), 5);
579
580        // get without key → BadRequest
581        assert!(matches!(&operations[0], Err(BatchError::BadRequest(_))));
582        // valid get
583        assert!(matches!(
584            &operations[1].as_ref().unwrap(),
585            Operation::Get(g) if g.key == "valid"
586        ));
587        // delete without key → BadRequest
588        assert!(matches!(&operations[2], Err(BatchError::BadRequest(_))));
589        // oversized insert → LimitExceeded
590        assert!(matches!(&operations[3], Err(BatchError::LimitExceeded(_))));
591        // valid delete still succeeds after prior errors
592        assert!(matches!(
593            &operations[4].as_ref().unwrap(),
594            Operation::Delete(d) if d.key == "valid"
595        ));
596    }
597
598    #[tokio::test]
599    async fn test_max_operations_limit_enforced() {
600        let mut body = String::new();
601        for i in 0..(MAX_BATCH_OPERATIONS + 1) {
602            let key = headers::encode_header_str(&format!("test{i}")).to_string();
603            body.push_str(&format!(
604                "--boundary\r\n\
605                 {HEADER_BATCH_OPERATION_KEY}: {key}\r\n\
606                 {HEADER_BATCH_OPERATION_KIND}: get\r\n\
607                 \r\n\
608                 \r\n"
609            ));
610        }
611        body.push_str("--boundary--\r\n");
612
613        let request = Request::builder()
614            .header(CONTENT_TYPE, "multipart/form-data; boundary=boundary")
615            .body(Body::from(body))
616            .unwrap();
617
618        let batch_request = BatchOperationStream::from_request(request, &())
619            .await
620            .unwrap();
621        let operations: Vec<_> = batch_request.operations.collect().await;
622
623        assert_eq!(operations.len(), MAX_BATCH_OPERATIONS + 1);
624        assert!(matches!(
625            &operations[MAX_BATCH_OPERATIONS],
626            Err(BatchError::LimitExceeded(_))
627        ));
628    }
629
630    #[tokio::test]
631    async fn test_head_operation() {
632        let key = headers::encode_header_str("head-key");
633        let body = format!(
634            "--boundary\r\n\
635             {HEADER_BATCH_OPERATION_KEY}: {key}\r\n\
636             {HEADER_BATCH_OPERATION_KIND}: head\r\n\
637             \r\n\
638             \r\n\
639             --boundary--\r\n",
640        );
641
642        let request = Request::builder()
643            .header(CONTENT_TYPE, "multipart/form-data; boundary=boundary")
644            .body(Body::from(body))
645            .unwrap();
646
647        let batch_request = BatchOperationStream::from_request(request, &())
648            .await
649            .unwrap();
650        let operations: Vec<_> = batch_request.operations.collect().await;
651        assert_eq!(operations.len(), 1);
652
653        let Operation::Head(head_op) = &operations[0].as_ref().unwrap() else {
654            panic!("expected head operation");
655        };
656        assert_eq!(head_op.key, "head-key");
657    }
658
659    #[tokio::test]
660    async fn test_head_without_key_is_error() {
661        let body = format!(
662            "--boundary\r\n\
663             {HEADER_BATCH_OPERATION_KIND}: head\r\n\
664             \r\n\
665             \r\n\
666             --boundary--\r\n",
667        );
668
669        let request = Request::builder()
670            .header(CONTENT_TYPE, "multipart/form-data; boundary=boundary")
671            .body(Body::from(body))
672            .unwrap();
673
674        let batch_request = BatchOperationStream::from_request(request, &())
675            .await
676            .unwrap();
677        let operations: Vec<_> = batch_request.operations.collect().await;
678        assert_eq!(operations.len(), 1);
679        assert!(matches!(&operations[0], Err(BatchError::BadRequest(_))));
680    }
681
682    // --- JSON layout -------------------------------------------------------------------------
683
684    const END: &str = "--boundary--\r\n";
685
686    fn descriptor(name: &str, json: &str) -> String {
687        format!("--boundary\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n{json}\r\n")
688    }
689
690    fn payload(name: &str, content_type: Option<&str>, body: &str) -> String {
691        let content_type = content_type
692            .map(|ct| format!("Content-Type: {ct}\r\n"))
693            .unwrap_or_default();
694        format!(
695            "--boundary\r\nContent-Disposition: form-data; name=\"{name}\"; filename=\"blob\"\r\n\
696             {content_type}\r\n{body}\r\n"
697        )
698    }
699
700    async fn parse(body: String) -> (BatchFormat, Vec<Result<Operation, BatchError>>) {
701        let request = Request::builder()
702            .extension(RequestTime(Timestamp::UNIX_EPOCH))
703            .header(CONTENT_TYPE, "multipart/form-data; boundary=boundary")
704            .body(Body::from(body))
705            .unwrap();
706        let stream = BatchOperationStream::from_request(request, &())
707            .await
708            .unwrap();
709        (stream.format, stream.operations.collect().await)
710    }
711
712    #[track_caller]
713    fn insert(item: &Result<Operation, BatchError>) -> &Insert {
714        match item {
715            Ok(Operation::Insert(insert)) => insert,
716            other => panic!("expected insert, got {other:?}"),
717        }
718    }
719
720    #[tokio::test]
721    async fn test_detects_layout() {
722        let header = format!(
723            "--boundary\r\n{HEADER_BATCH_OPERATION_KIND}: get\r\n\
724             {HEADER_BATCH_OPERATION_KEY}: a\r\n\r\n\r\n{END}"
725        );
726        assert_eq!(parse(header).await.0, BatchFormat::Headers);
727
728        let json = descriptor("0", r#"{"kind":"get","key":"a"}"#) + END;
729        assert_eq!(parse(json).await.0, BatchFormat::Json);
730
731        let (format, operations) = parse(END.to_owned()).await;
732        assert_eq!(format, BatchFormat::Json);
733        assert!(operations.is_empty());
734    }
735
736    #[tokio::test]
737    async fn test_json_all_kinds() {
738        let body = [
739            descriptor("0", r#"{"kind":"get","key":"a/résumé.png"}"#),
740            descriptor("1", r#"{"kind":"head","key":"b","ignored":true}"#),
741            // `metadata` is ignored for other kinds.
742            descriptor("2", r#"{"kind":"delete","key":"c","metadata":{"origin":5}}"#),
743            descriptor(
744                "3",
745                r#"{"kind":"insert","key":"d","metadata":{"content_type":"image/png","expiration":"ttl:1h"}}"#,
746            ),
747            payload("3.payload", None, "png"),
748            descriptor("4", r#"{"kind":"insert"}"#),
749            payload("4.payload", None, "keyless"),
750        ]
751        .concat()
752            + END;
753
754        let (_, operations) = parse(body).await;
755        assert_eq!(operations.len(), 5);
756        assert!(matches!(&operations[0], Ok(Operation::Get(op)) if op.key == "a/résumé.png"));
757        assert!(matches!(&operations[1], Ok(Operation::Head(op)) if op.key == "b"));
758        assert!(matches!(&operations[2], Ok(Operation::Delete(op)) if op.key == "c"));
759
760        let inserted = insert(&operations[3]);
761        assert_eq!(inserted.key.as_deref(), Some("d"));
762        assert_eq!(inserted.payload.as_ref(), b"png");
763        assert_eq!(inserted.metadata.content_type, "image/png");
764        assert_eq!(
765            inserted.metadata.time_expires,
766            Some(Timestamp::UNIX_EPOCH + Duration::from_hours(1))
767        );
768
769        assert_eq!(insert(&operations[4]).key, None);
770    }
771
772    #[tokio::test]
773    async fn test_json_content_type_resolution() {
774        let body = [
775            // Metadata wins over the payload part.
776            descriptor(
777                "0",
778                r#"{"kind":"insert","metadata":{"content_type":"image/png"}}"#,
779            ),
780            payload("0.payload", Some("text/plain"), "a"),
781            // Without metadata, the payload part's type is used.
782            descriptor("1", r#"{"kind":"insert"}"#),
783            payload("1.payload", Some("text/plain"), "b"),
784            // Encoded payloads do not use the payload part's type.
785            descriptor(
786                "2",
787                r#"{"kind":"insert","metadata":{"content_encoding":"gzip"}}"#,
788            ),
789            payload("2.payload", Some("text/plain"), "c"),
790        ]
791        .concat()
792            + END;
793
794        let (_, operations) = parse(body).await;
795        let content_types: Vec<_> = operations
796            .iter()
797            .map(|item| insert(item).metadata.content_type.as_ref())
798            .collect();
799        assert_eq!(
800            content_types,
801            ["image/png", "text/plain", "application/octet-stream"]
802        );
803    }
804
805    #[tokio::test]
806    async fn test_json_invalid_operations() {
807        let body = [
808            // The index is the position, so "1" at position 0 fails.
809            descriptor("1", r#"{"kind":"get","key":"a"}"#),
810            descriptor("1", "not json"),
811            descriptor("2", r#"{"kind":"fetch","key":"a"}"#),
812            descriptor("3", r#"{"kind":"get"}"#),
813            descriptor(
814                "4",
815                r#"{"kind":"insert","metadata":{"expiration":"forever"}}"#,
816            ),
817            payload("4.payload", None, "x"),
818            descriptor("5", r#"{"kind":"get","key":"ok"}"#),
819        ]
820        .concat()
821            + END;
822
823        let (_, operations) = parse(body).await;
824        assert_eq!(operations.len(), 6);
825        for item in &operations[..5] {
826            let error = item.as_ref().expect_err("expected an error");
827            assert_eq!(error.status(), StatusCode::BAD_REQUEST);
828        }
829        assert!(operations[5].is_ok());
830    }
831
832    #[tokio::test]
833    async fn test_json_payload_pairing() {
834        let body = [
835            // The payload after a failed descriptor is skipped.
836            descriptor("x", r#"{"kind":"get","key":"a"}"#),
837            payload("0.payload", None, r#"{"kind":"get","key":"b"}"#),
838            // An insert without its payload fails; the next part is the next descriptor.
839            descriptor("1", r#"{"kind":"insert","key":"c"}"#),
840            descriptor("2", r#"{"kind":"get","key":"d"}"#),
841            // Payloads that do not follow their insert are skipped.
842            payload("other.payload", None, "stray"),
843            descriptor("3", r#"{"kind":"insert","key":"e"}"#),
844            payload("3.payload", None, "ok"),
845        ]
846        .concat()
847            + END;
848
849        let (_, operations) = parse(body).await;
850        assert_eq!(operations.len(), 4);
851        assert!(operations[0].is_err());
852        assert!(operations[1].is_err());
853        assert!(matches!(&operations[2], Ok(Operation::Get(op)) if op.key == "d"));
854        assert_eq!(insert(&operations[3]).payload.as_ref(), b"ok");
855    }
856
857    #[tokio::test]
858    async fn test_json_broken_body() {
859        let get = descriptor("0", r#"{"kind":"get","key":"a"}"#);
860        let broken = "--boundary\r\nnot a header line\r\n\r\n{}\r\n";
861        let cases = [
862            // The error goes to the insert waiting for its payload.
863            get.clone() + &descriptor("1", r#"{"kind":"insert"}"#) + broken,
864            // Otherwise it goes to the next index, and later parts are never read.
865            get.clone() + broken + &descriptor("2", r#"{"kind":"get","key":"c"}"#) + END,
866        ];
867
868        for body in cases {
869            let (_, operations) = parse(body).await;
870            assert_eq!(operations.len(), 2);
871            assert!(operations[0].is_ok());
872            assert!(matches!(operations[1], Err(BatchError::Multipart(_))));
873        }
874    }
875
876    #[tokio::test]
877    async fn test_json_size_limits() {
878        let large_payload = "x".repeat(MAX_BATCH_PART_SIZE + 1);
879        let large_key = "k".repeat(MAX_DESCRIPTOR_SIZE);
880        let body = [
881            descriptor("0", r#"{"kind":"insert"}"#),
882            payload("0.payload", None, &large_payload),
883            descriptor("1", &format!(r#"{{"kind":"get","key":"{large_key}"}}"#)),
884        ]
885        .concat()
886            + END;
887
888        let (_, operations) = parse(body).await;
889        assert!(matches!(
890            operations[..],
891            [
892                Err(BatchError::LimitExceeded(_)),
893                Err(BatchError::LimitExceeded(_))
894            ]
895        ));
896    }
897
898    #[tokio::test]
899    async fn test_json_operation_limit_counts_descriptors() {
900        let mut body = String::new();
901        for index in 0..=MAX_BATCH_OPERATIONS {
902            body += &descriptor(&index.to_string(), r#"{"kind":"insert"}"#);
903            body += &payload(&batch::payload_part_name(index), None, "x");
904        }
905        body += END;
906
907        let (_, operations) = parse(body).await;
908        assert_eq!(operations.len(), MAX_BATCH_OPERATIONS + 1);
909        assert!(operations[..MAX_BATCH_OPERATIONS].iter().all(Result::is_ok));
910        assert!(matches!(
911            operations[MAX_BATCH_OPERATIONS],
912            Err(BatchError::LimitExceeded(_))
913        ));
914    }
915}