1use std::collections::{HashMap, HashSet};
2use std::fmt;
3use std::io;
4use std::pin::Pin;
5use std::sync::Arc;
6use std::task::{Context, Poll};
7
8use futures_util::{Stream, StreamExt as _};
9use multer::Field;
10use objectstore_types::batch::{MAX_BATCH_OPERATIONS, MAX_BATCH_PART_SIZE};
11use objectstore_types::headers;
12use objectstore_types::metadata::{Compression, Metadata};
13use reqwest::header::{CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue};
14use reqwest::multipart::Part;
15
16use crate::error::Error;
17use crate::instrument::OperationSpan;
18use crate::metadata;
19use crate::put::{CompressionMode, PutBody};
20use crate::response::ResponseExt as _;
21use crate::{
22 DeleteBuilder, DeleteResponse, GetBuilder, GetResponse, HeadBuilder, HeadResponse, ObjectKey,
23 PutBuilder, PutResponse, Session, get, put,
24};
25
26const HEADER_BATCH_OPERATION_KEY: &str = "x-sn-batch-operation-key";
27const HEADER_BATCH_OPERATION_KIND: &str = "x-sn-batch-operation-kind";
28const HEADER_BATCH_OPERATION_INDEX: &str = "x-sn-batch-operation-index";
29const HEADER_BATCH_OPERATION_STATUS: &str = "x-sn-batch-operation-status";
30
31const DEFAULT_INDIVIDUAL_CONCURRENCY: usize = 5;
35
36const DEFAULT_BATCH_CONCURRENCY: usize = 3;
40
41const MAX_BATCH_BODY_SIZE: u64 = 100 * 1024 * 1024; #[derive(Debug)]
49pub struct ManyBuilder {
50 session: Session,
51 operations: Vec<BatchOperation>,
52 max_individual_concurrency: Option<usize>,
53 max_batch_concurrency: Option<usize>,
54}
55
56impl Session {
57 pub fn many(&self) -> ManyBuilder {
62 ManyBuilder {
63 session: self.clone(),
64 operations: vec![],
65 max_individual_concurrency: None,
66 max_batch_concurrency: None,
67 }
68 }
69}
70
71#[derive(Debug)]
72#[allow(clippy::large_enum_variant)]
73enum BatchOperation {
74 Get {
75 key: ObjectKey,
76 decompress: bool,
77 accept_encoding: Vec<Compression>,
78 },
79 Insert {
80 key: Option<ObjectKey>,
81 metadata: Metadata,
82 compression: Option<CompressionMode>,
83 body: PutBody,
84 },
85 Delete {
86 key: ObjectKey,
87 },
88 Head {
89 key: ObjectKey,
90 },
91}
92
93impl From<GetBuilder> for BatchOperation {
94 fn from(value: GetBuilder) -> Self {
95 let GetBuilder {
96 key,
97 decompress,
98 accept_encoding,
99 session: _session,
100 } = value;
101 BatchOperation::Get {
102 key,
103 decompress,
104 accept_encoding,
105 }
106 }
107}
108
109impl From<PutBuilder> for BatchOperation {
110 fn from(value: PutBuilder) -> Self {
111 let PutBuilder {
112 key,
113 metadata,
114 compression,
115 body,
116 session: _session,
117 resumable_threshold_bytes: _,
118 } = value;
119 BatchOperation::Insert {
120 key,
121 metadata,
122 compression,
123 body,
124 }
125 }
126}
127
128impl From<DeleteBuilder> for BatchOperation {
129 fn from(value: DeleteBuilder) -> Self {
130 let DeleteBuilder {
131 key,
132 session: _session,
133 } = value;
134 BatchOperation::Delete { key }
135 }
136}
137
138impl From<HeadBuilder> for BatchOperation {
139 fn from(value: HeadBuilder) -> Self {
140 let HeadBuilder {
141 key,
142 session: _session,
143 } = value;
144 BatchOperation::Head { key }
145 }
146}
147
148impl BatchOperation {
149 async fn into_part(self) -> crate::Result<Part> {
150 match self {
151 BatchOperation::Get { key, .. } => {
152 let headers = operation_headers("get", Some(&key));
153 Ok(Part::text("").headers(headers))
154 }
155 BatchOperation::Insert {
156 key,
157 mut metadata,
158 compression,
159 body,
160 } => {
161 let mut headers = operation_headers("insert", key.as_deref());
162 metadata.compression = compression.map(CompressionMode::compression);
163 headers.extend(metadata.to_headers("")?);
164
165 let encoded = put::encode_body(body, compression).await?;
168 let part = match encoded.length {
169 Some(length) => Part::stream_with_length(encoded.body, length),
170 None => Part::stream(encoded.body),
171 };
172 Ok(part.headers(headers))
173 }
174 BatchOperation::Delete { key } => {
175 let headers = operation_headers("delete", Some(&key));
176 Ok(Part::text("").headers(headers))
177 }
178 BatchOperation::Head { key } => {
179 let headers = operation_headers("head", Some(&key));
180 Ok(Part::text("").headers(headers))
181 }
182 }
183 }
184}
185
186fn operation_headers(operation: &str, key: Option<&str>) -> HeaderMap {
187 let mut headers = HeaderMap::new();
188 headers.insert(
189 HeaderName::from_static(HEADER_BATCH_OPERATION_KIND),
190 HeaderValue::from_str(operation).expect("operation kind is always a valid header value"),
191 );
192 if let Some(key) = key {
193 headers.insert(
194 HeaderName::from_static(HEADER_BATCH_OPERATION_KEY),
195 headers::encode_header_value(key),
196 );
197 }
198 headers
199}
200
201#[derive(Debug)]
203pub enum OperationResult {
204 Get(ObjectKey, Result<Option<GetResponse>, Error>),
208 Put(ObjectKey, Result<PutResponse, Error>),
210 Delete(ObjectKey, Result<DeleteResponse, Error>),
212 Head(ObjectKey, Result<HeadResponse, Error>),
216 Error(Error),
222}
223
224enum OperationContext {
226 Get {
227 key: ObjectKey,
228 decompress: bool,
229 accept_encoding: Vec<Compression>,
230 },
231 Insert {
232 key: Option<ObjectKey>,
233 },
234 Delete {
235 key: ObjectKey,
236 },
237 Head {
238 key: ObjectKey,
239 },
240}
241
242impl From<&BatchOperation> for OperationContext {
243 fn from(op: &BatchOperation) -> Self {
244 match op {
245 BatchOperation::Get {
246 key,
247 decompress,
248 accept_encoding,
249 } => OperationContext::Get {
250 key: key.clone(),
251 decompress: *decompress,
252 accept_encoding: accept_encoding.clone(),
253 },
254 BatchOperation::Insert { key, .. } => OperationContext::Insert { key: key.clone() },
255 BatchOperation::Delete { key } => OperationContext::Delete { key: key.clone() },
256 BatchOperation::Head { key } => OperationContext::Head { key: key.clone() },
257 }
258 }
259}
260
261impl OperationContext {
262 fn key(&self) -> Option<&str> {
263 match self {
264 OperationContext::Get { key, .. }
265 | OperationContext::Delete { key }
266 | OperationContext::Head { key } => Some(key),
267 OperationContext::Insert { key } => key.as_deref(),
268 }
269 }
270}
271
272#[derive(Debug)]
274enum Classified {
275 Batchable(BatchOperation, u64),
277 Individual(BatchOperation),
279 Failed(OperationResult),
281}
282
283fn error_result(ctx: OperationContext, error: Error) -> OperationResult {
285 let key = ctx.key().unwrap_or("<unknown>").to_owned();
286 match ctx {
287 OperationContext::Get { .. } => OperationResult::Get(key, Err(error)),
288 OperationContext::Insert { .. } => OperationResult::Put(key, Err(error)),
289 OperationContext::Delete { .. } => OperationResult::Delete(key, Err(error)),
290 OperationContext::Head { .. } => OperationResult::Head(key, Err(error)),
291 }
292}
293
294impl OperationResult {
295 async fn from_field(
296 field: Field<'_>,
297 context_map: &HashMap<usize, OperationContext>,
298 ) -> (Option<usize>, Self) {
299 match Self::try_from_field(field, context_map).await {
300 Ok((index, result)) => (Some(index), result),
301 Err(e) => (None, OperationResult::Error(e)),
302 }
303 }
304
305 async fn try_from_field(
306 field: Field<'_>,
307 context_map: &HashMap<usize, OperationContext>,
308 ) -> Result<(usize, Self), Error> {
309 let mut headers = field.headers().clone();
310
311 let index: usize = headers
312 .remove(HEADER_BATCH_OPERATION_INDEX)
313 .and_then(|v| v.to_str().ok().and_then(|s| s.parse().ok()))
314 .ok_or_else(|| {
315 Error::MalformedResponse(format!(
316 "missing or invalid {HEADER_BATCH_OPERATION_INDEX} header"
317 ))
318 })?;
319
320 let status: u16 = headers
321 .remove(HEADER_BATCH_OPERATION_STATUS)
322 .and_then(|v| {
323 v.to_str().ok().and_then(|s| {
324 s.split_once(' ')
327 .map(|(code, _)| code)
328 .unwrap_or(s)
329 .parse()
330 .ok()
331 })
332 })
333 .ok_or_else(|| {
334 Error::MalformedResponse(format!(
335 "missing or invalid {HEADER_BATCH_OPERATION_STATUS} header"
336 ))
337 })?;
338
339 let ctx = context_map.get(&index).ok_or_else(|| {
340 Error::MalformedResponse(format!(
341 "response references unknown operation index {index}"
342 ))
343 })?;
344
345 let key = headers
347 .remove(HEADER_BATCH_OPERATION_KEY)
348 .and_then(|v| headers::decode_header_value(&v).ok())
349 .or_else(|| ctx.key().map(str::to_owned));
350
351 let body = field.bytes().await?;
352
353 let is_error = status >= 400
354 && !(matches!(
355 ctx,
356 OperationContext::Get { .. } | OperationContext::Head { .. }
357 ) && status == 404);
358
359 let key = match key {
364 Some(key) => key,
365 None if is_error => "<unknown>".to_owned(),
366 None => {
367 return Err(Error::MalformedResponse(format!(
368 "missing or invalid {HEADER_BATCH_OPERATION_KEY} header"
369 )));
370 }
371 };
372 if is_error {
373 let message = String::from_utf8_lossy(&body).into_owned();
374 let error = Error::OperationFailure { status, message };
375
376 return Ok((
377 index,
378 match ctx {
379 OperationContext::Get { .. } => OperationResult::Get(key, Err(error)),
380 OperationContext::Insert { .. } => OperationResult::Put(key, Err(error)),
381 OperationContext::Delete { .. } => OperationResult::Delete(key, Err(error)),
382 OperationContext::Head { .. } => OperationResult::Head(key, Err(error)),
383 },
384 ));
385 }
386
387 let result = match ctx {
388 OperationContext::Get {
389 decompress,
390 accept_encoding,
391 ..
392 } => {
393 if status == 404 {
394 OperationResult::Get(key, Ok(None))
395 } else {
396 let mut metadata = Metadata::from_headers(&headers, "")?;
397
398 let stream =
399 futures_util::stream::once(async move { Ok::<_, io::Error>(body) }).boxed();
400 let stream =
401 get::maybe_decompress(stream, &mut metadata, *decompress, accept_encoding);
402
403 OperationResult::Get(key, Ok(Some(GetResponse { metadata, stream })))
404 }
405 }
406 OperationContext::Insert { .. } => {
407 OperationResult::Put(key.clone(), Ok(PutResponse { key }))
408 }
409 OperationContext::Delete { .. } => OperationResult::Delete(key, Ok(())),
410 OperationContext::Head { .. } => {
411 if status == 404 {
412 OperationResult::Head(key, Ok(None))
413 } else {
414 let metadata = Metadata::from_headers(&headers, "")?;
415 OperationResult::Head(key, Ok(Some(metadata)))
416 }
417 }
418 };
419 Ok((index, result))
420 }
421}
422
423pub struct OperationResults(Pin<Box<dyn Stream<Item = OperationResult> + Send>>);
425
426impl fmt::Debug for OperationResults {
427 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
428 f.write_str("OperationResults([Stream])")
429 }
430}
431
432impl Stream for OperationResults {
433 type Item = OperationResult;
434
435 fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
436 self.0.as_mut().poll_next(cx)
437 }
438}
439
440impl OperationResults {
441 pub async fn error_for_failures(mut self) -> crate::Result<(), impl Iterator<Item = Error>> {
446 let mut errs = Vec::new();
447 while let Some(res) = self.next().await {
448 match res {
449 OperationResult::Get(_, get) => {
450 if let Err(e) = get {
451 errs.push(e);
452 }
453 }
454 OperationResult::Put(_, put) => {
455 if let Err(e) = put {
456 errs.push(e);
457 }
458 }
459 OperationResult::Delete(_, delete) => {
460 if let Err(e) = delete {
461 errs.push(e);
462 }
463 }
464 OperationResult::Head(_, head) => {
465 if let Err(e) = head {
466 errs.push(e);
467 }
468 }
469 OperationResult::Error(error) => errs.push(error),
470 }
471 }
472 if errs.is_empty() {
473 return Ok(());
474 }
475 Err(errs.into_iter())
476 }
477}
478
479async fn send_batch(
480 session: &Session,
481 operations: Vec<BatchOperation>,
482) -> crate::Result<Vec<OperationResult>> {
483 let mut context_map: HashMap<usize, OperationContext> = operations
484 .iter()
485 .enumerate()
486 .map(|(idx, op)| (idx, OperationContext::from(op)))
487 .collect();
488 let num_operations = operations.len();
489
490 let mut form = reqwest::multipart::Form::new();
491 for op in operations {
492 let part = op.into_part().await?;
493 form = form.part("part", part);
494 }
495
496 let request = session.batch_request()?.multipart(form);
497 let response = request.send().await?.error_for_status_and_drain().await?;
498
499 let boundary = response
500 .headers()
501 .get(CONTENT_TYPE)
502 .and_then(|v| v.to_str().ok())
503 .ok_or_else(|| Error::MalformedResponse("missing Content-Type header".to_owned()))
504 .map(multer::parse_boundary)??;
505
506 let byte_stream = response.bytes_stream().map(|r| r.map_err(io::Error::other));
510 let mut multipart = multer::Multipart::new(byte_stream, boundary);
511
512 let mut results = Vec::new();
513 let mut seen_indices = HashSet::new();
514 while let Some(field) = multipart.next_field().await? {
515 let (index, result) = OperationResult::from_field(field, &context_map).await;
516 if let Some(idx) = index {
517 seen_indices.insert(idx);
518 }
519 results.push(result);
520 }
521
522 for idx in 0..num_operations {
523 if !seen_indices.contains(&idx) {
524 let error = Error::MalformedResponse(format!(
525 "server did not return a response for operation at index {idx}"
526 ));
527 let result = match context_map.remove(&idx) {
528 Some(ctx) => error_result(ctx, error),
529 None => OperationResult::Error(error),
530 };
531 results.push(result);
532 }
533 }
534
535 Ok(results)
536}
537
538fn compress_bound(compression: Compression, size: usize) -> usize {
540 match compression {
541 Compression::Zstd => zstd_safe::compress_bound(size),
542 Compression::Gzip => deflate_bound(size) + GZIP_FRAMING_SIZE,
543 }
544}
545
546const GZIP_FRAMING_SIZE: usize = 18;
548
549fn deflate_bound(size: usize) -> usize {
551 size + (size >> 12) + (size >> 14) + (size >> 25) + 7
556}
557
558fn classify_fail(key: Option<ObjectKey>, error: Error) -> Classified {
559 Classified::Failed(OperationResult::Put(
560 key.unwrap_or_else(|| "<unknown>".to_owned()),
561 Err(error),
562 ))
563}
564
565async fn classify(op: BatchOperation) -> Classified {
570 match op {
571 BatchOperation::Insert {
572 key,
573 mut metadata,
574 compression,
575 body,
576 } => {
577 if let Err(err) = metadata::normalize_keys(&mut metadata) {
579 return classify_fail(key, err);
580 }
581
582 let size = match &body {
583 PutBody::Buffer(bytes) => Some(bytes.len() as u64),
584 PutBody::File(file) => match file.metadata().await {
585 Ok(meta) => Some(meta.len()),
586 Err(err) => return classify_fail(key, err.into()),
587 },
588 PutBody::Path(path) => match tokio::fs::metadata(path).await {
589 Ok(meta) => Some(meta.len()),
590 Err(err) => return classify_fail(key, err.into()),
591 },
592 PutBody::Stream(_) => None,
594 };
595
596 let size = match (compression, size) {
599 (Some(CompressionMode::Compress(compression)), Some(size)) => usize::try_from(size)
600 .ok()
601 .map(|size| compress_bound(compression, size)),
602 (Some(CompressionMode::Precompressed(_)) | None, Some(size)) => {
603 usize::try_from(size).ok()
604 }
605 (_, None) => None,
606 };
607
608 let op = BatchOperation::Insert {
609 key,
610 metadata,
611 compression,
612 body,
613 };
614
615 match size {
616 Some(s) if s <= MAX_BATCH_PART_SIZE => Classified::Batchable(op, s as u64),
617 _ => Classified::Individual(op),
618 }
619 }
620 other => Classified::Batchable(other, 0),
621 }
622}
623
624async fn partition(
628 operations: Vec<BatchOperation>,
629) -> (
630 Vec<(BatchOperation, u64)>,
631 Vec<BatchOperation>,
632 Vec<OperationResult>,
633) {
634 let classified = futures_util::future::join_all(operations.into_iter().map(classify)).await;
635 let mut batchable = Vec::new();
636 let mut individual = Vec::new();
637 let mut failed = Vec::new();
638 for item in classified {
639 match item {
640 Classified::Batchable(op, size) => batchable.push((op, size)),
641 Classified::Individual(op) => individual.push(op),
642 Classified::Failed(result) => failed.push(result),
643 }
644 }
645 (batchable, individual, failed)
646}
647
648async fn execute_individual(op: BatchOperation, session: &Session) -> OperationResult {
650 match op {
651 BatchOperation::Get {
652 key,
653 decompress,
654 accept_encoding,
655 } => {
656 let get = GetBuilder {
657 session: session.clone(),
658 key: key.clone(),
659 decompress,
660 accept_encoding,
661 };
662 OperationResult::Get(key, get.send().await)
663 }
664 BatchOperation::Insert {
665 key,
666 metadata,
667 compression,
668 body,
669 } => {
670 let error_key = key.clone().unwrap_or_else(|| "<unknown>".to_owned());
671 let put = PutBuilder {
672 resumable_threshold_bytes: None,
674 session: session.clone(),
675 metadata,
676 compression,
677 key,
678 body,
679 };
680 match put.send().await {
681 Ok(response) => OperationResult::Put(response.key.clone(), Ok(response)),
682 Err(err) => OperationResult::Put(error_key, Err(err)),
683 }
684 }
685 BatchOperation::Delete { key } => {
686 let delete = DeleteBuilder {
687 session: session.clone(),
688 key: key.clone(),
689 };
690 OperationResult::Delete(key, delete.send().await)
691 }
692 BatchOperation::Head { key } => {
693 let head = HeadBuilder {
694 session: session.clone(),
695 key: key.clone(),
696 };
697 OperationResult::Head(key, head.send().await)
698 }
699 }
700}
701
702async fn execute_batch(operations: Vec<BatchOperation>, session: &Session) -> Vec<OperationResult> {
706 let contexts: Vec<_> = operations.iter().map(OperationContext::from).collect();
707 let span = OperationSpan::batch(session, operations.len());
708 match span.instrument(send_batch(session, operations)).await {
709 Ok(results) => results,
710 Err(e) => {
711 let shared = Arc::new(e);
712 contexts
713 .into_iter()
714 .map(|ctx| error_result(ctx, Error::Batch(shared.clone())))
715 .collect()
716 }
717 }
718}
719
720fn iter_batches(ops: Vec<(BatchOperation, u64)>) -> impl Iterator<Item = Vec<BatchOperation>> {
725 let mut remaining = ops.into_iter().peekable();
726
727 std::iter::from_fn(move || {
728 remaining.peek()?;
729 let mut batch_size = 0;
730 let mut batch = Vec::new();
731
732 while let Some((_, op_size)) = remaining.peek() {
733 if batch.len() >= MAX_BATCH_OPERATIONS
734 || (!batch.is_empty() && batch_size + op_size > MAX_BATCH_BODY_SIZE)
735 {
736 break;
737 }
738
739 let (op, op_size) = remaining.next().expect("peeked above");
740 batch_size += op_size;
741 batch.push(op);
742 }
743
744 Some(batch)
745 })
746}
747
748impl ManyBuilder {
749 pub async fn send(self) -> OperationResults {
753 let session = self.session;
754 let individual_concurrency = self
755 .max_individual_concurrency
756 .unwrap_or(DEFAULT_INDIVIDUAL_CONCURRENCY)
757 .max(1);
758 let batch_concurrency = self
759 .max_batch_concurrency
760 .unwrap_or(DEFAULT_BATCH_CONCURRENCY)
761 .max(1);
762
763 let (batchable, individual, failed) = partition(self.operations).await;
765
766 let individual_results = futures_util::stream::iter(individual)
768 .map({
769 let session = session.clone();
770 move |op| {
771 let session = session.clone();
772 async move { execute_individual(op, &session).await }
773 }
774 })
775 .buffer_unordered(individual_concurrency);
776
777 let batch_results = futures_util::stream::iter(iter_batches(batchable))
779 .map(move |chunk| {
780 let session = session.clone();
781 async move { execute_batch(chunk, &session).await }
782 })
783 .buffer_unordered(batch_concurrency)
784 .flat_map(futures_util::stream::iter);
785
786 let results = futures_util::stream::iter(failed)
787 .chain(individual_results)
788 .chain(batch_results);
789
790 OperationResults(results.boxed())
791 }
792
793 pub fn max_individual_concurrency(mut self, concurrency: usize) -> Self {
799 self.max_individual_concurrency = Some(concurrency);
800 self
801 }
802
803 pub fn max_batch_concurrency(mut self, concurrency: usize) -> Self {
809 self.max_batch_concurrency = Some(concurrency);
810 self
811 }
812
813 #[allow(private_bounds)]
823 pub fn push<B: Into<BatchOperation>>(mut self, builder: B) -> Self {
824 self.operations.push(builder.into());
825 self
826 }
827}
828
829#[cfg(test)]
830mod tests {
831 use super::*;
832
833 fn op(size: u64) -> (BatchOperation, u64) {
835 (
836 BatchOperation::Delete {
837 key: "k".to_owned(),
838 },
839 size,
840 )
841 }
842
843 fn batch_sizes(batches: &[Vec<BatchOperation>]) -> Vec<usize> {
844 batches.iter().map(Vec::len).collect()
845 }
846
847 fn batches(ops: Vec<(BatchOperation, u64)>) -> Vec<Vec<BatchOperation>> {
848 iter_batches(ops).collect()
849 }
850
851 fn put_with_compression(size: usize, compression: CompressionMode) -> BatchOperation {
852 BatchOperation::Insert {
853 key: Some("k".to_owned()),
854 metadata: Metadata::default(),
855 compression: Some(compression),
856 body: PutBody::Buffer(vec![0; size].into()),
857 }
858 }
859
860 fn put_with_zstd(size: usize) -> BatchOperation {
861 put_with_compression(size, CompressionMode::Compress(Compression::Zstd))
862 }
863
864 #[tokio::test]
865 async fn zstd_put_at_limit_is_batchable() {
866 let size = 1_044_496;
867 let post_compression = zstd_safe::compress_bound(size);
868 assert!(post_compression == MAX_BATCH_PART_SIZE);
869
870 core::assert_matches!(
871 classify(put_with_zstd(size)).await,
872 Classified::Batchable(_, s) if s == post_compression as u64
873 );
874 }
875
876 #[tokio::test]
877 async fn zstd_put_above_limit_is_individual() {
878 let size = 1_044_497;
879 let post_compression = zstd_safe::compress_bound(size);
880 assert!(post_compression > MAX_BATCH_PART_SIZE);
881
882 core::assert_matches!(
883 classify(put_with_zstd(size)).await,
884 Classified::Individual(_)
885 );
886 }
887
888 #[tokio::test]
889 async fn gzip_put_bound() {
890 let size = 1_000_000;
891 let op = put_with_compression(size, CompressionMode::Compress(Compression::Gzip));
892 std::assert_matches!(
893 classify(op).await,
894 Classified::Batchable(_, s) if s == compress_bound(Compression::Gzip, size) as u64
895 );
896 }
897
898 #[tokio::test]
899 async fn precompressed_put_uses_exact_size() {
900 let size = MAX_BATCH_PART_SIZE;
903 assert!(zstd_safe::compress_bound(size) > MAX_BATCH_PART_SIZE);
904
905 let op = put_with_compression(size, CompressionMode::Precompressed(Compression::Zstd));
906 core::assert_matches!(
907 classify(op).await,
908 Classified::Batchable(_, s) if s == size as u64
909 );
910 }
911
912 #[test]
913 fn iter_batches_empty() {
914 assert!(batches(vec![]).is_empty());
915 }
916
917 #[test]
918 fn iter_batches_single_batch_count_limit() {
919 let ops: Vec<_> = (0..1000).map(|_| op(1)).collect();
921 assert_eq!(batch_sizes(&batches(ops)), vec![1000]);
922 }
923
924 #[test]
925 fn iter_batches_splits_on_count_limit() {
926 let ops: Vec<_> = (0..1001).map(|_| op(1)).collect();
928 assert_eq!(batch_sizes(&batches(ops)), vec![1000, 1]);
929 }
930
931 #[test]
932 fn iter_batches_exactly_at_size_limit() {
933 let ops: Vec<_> = (0..100).map(|_| op(1024 * 1024)).collect();
935 assert_eq!(batch_sizes(&batches(ops)), vec![100]);
936 }
937
938 #[test]
939 fn iter_batches_splits_on_size_limit() {
940 let ops: Vec<_> = (0..101).map(|_| op(1024 * 1024)).collect();
942 assert_eq!(batch_sizes(&batches(ops)), vec![100, 1]);
943 }
944
945 #[test]
946 fn iter_batches_size_limit_hits_before_count_limit() {
947 let op_size = 600 * 1024;
949 let ops: Vec<_> = (0..200).map(|_| op(op_size)).collect();
950 let result = batches(ops);
951 let per_batch = (MAX_BATCH_BODY_SIZE / op_size) as usize;
953 assert!(result.len() > 1, "expected multiple batches");
954 for batch in &result[..result.len() - 1] {
955 assert_eq!(batch.len(), per_batch);
956 }
957 }
958}