1use std::fmt;
2use std::io::{self, Cursor};
3use std::path::PathBuf;
4use std::{borrow::Cow, collections::BTreeMap};
5
6use async_compression::tokio::bufread::{GzipEncoder, ZstdEncoder};
7use bytes::Bytes;
8use futures_util::StreamExt;
9use objectstore_types::metadata::Metadata;
10use reqwest::Body;
11use reqwest::header::CONTENT_LENGTH;
12use serde::Deserialize;
13use tokio::fs::File;
14use tokio::io::{AsyncBufRead, AsyncRead, AsyncReadExt as _, AsyncSeekExt as _, BufReader};
15use tokio_util::io::{ReaderStream, StreamReader};
16
17pub use objectstore_types::metadata::{Compression, ExpirationPolicy};
18
19use crate::instrument::{OperationSpan, debug, record_field};
20use crate::metadata;
21use crate::response::ResponseExt as _;
22use crate::resumable::CreateResumableUploadBuilder;
23use crate::{ClientStream, ObjectKey, Session};
24
25mod automatic;
26mod payload;
27
28#[derive(Debug, Deserialize)]
30pub struct PutResponse {
31 pub key: ObjectKey,
33}
34
35pub(crate) enum PutBody {
36 Buffer(Bytes),
37 Stream(ClientStream),
38 File(File),
39 Path(PathBuf),
40}
41
42impl fmt::Debug for PutBody {
43 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
44 f.debug_tuple("PutBody").finish_non_exhaustive()
45 }
46}
47
48#[derive(Debug, Clone, Copy, PartialEq, Eq)]
53pub(crate) enum CompressionMode {
54 Compress(Compression),
56 Precompressed(Compression),
58}
59
60impl CompressionMode {
61 pub fn compression(self) -> Compression {
63 match self {
64 Self::Compress(compression) | Self::Precompressed(compression) => compression,
65 }
66 }
67}
68
69impl Session {
70 fn put_body(&self, body: PutBody) -> PutBuilder {
71 let metadata = Metadata {
72 expiration_policy: self.scope.usecase().expiration_policy(),
73 ..Default::default()
74 };
75
76 PutBuilder {
77 session: self.clone(),
78 metadata,
79 compression: self
80 .scope
81 .usecase()
82 .compression()
83 .map(CompressionMode::Compress),
84 key: None,
85 resumable_threshold_bytes: self.scope.usecase().resumable_threshold_bytes(),
86 body,
87 }
88 }
89
90 pub fn put(&self, body: impl Into<Bytes>) -> PutBuilder {
92 self.put_body(PutBody::Buffer(body.into()))
93 }
94
95 pub fn put_stream(&self, body: ClientStream) -> PutBuilder {
99 self.put_body(PutBody::Stream(body))
100 }
101
102 pub fn put_read<R>(&self, body: R) -> PutBuilder
106 where
107 R: AsyncRead + Send + Sync + 'static,
108 {
109 let stream = ReaderStream::new(body).boxed();
110 self.put_body(PutBody::Stream(stream))
111 }
112
113 pub fn put_file(&self, file: File) -> PutBuilder {
124 self.put_body(PutBody::File(file))
125 }
126
127 pub fn put_path(&self, path: impl Into<PathBuf>) -> PutBuilder {
144 self.put_body(PutBody::Path(path.into()))
145 }
146}
147
148#[derive(Debug)]
158pub struct PutBuilder {
159 pub(crate) session: Session,
160 pub(crate) metadata: Metadata,
161 pub(crate) compression: Option<CompressionMode>,
162 pub(crate) key: Option<ObjectKey>,
163 pub(crate) body: PutBody,
164 pub(crate) resumable_threshold_bytes: Option<u64>,
165}
166
167impl PutBuilder {
168 pub fn resumable_threshold_bytes(mut self, threshold: impl Into<Option<u64>>) -> Self {
182 self.resumable_threshold_bytes = threshold.into();
183 self
184 }
185
186 pub fn key(mut self, key: impl Into<ObjectKey>) -> Self {
191 self.key = Some(key.into()).filter(|k| !k.is_empty());
192 self
193 }
194
195 pub fn compress(mut self, compression: impl Into<Option<Compression>>) -> Self {
220 self.compression = compression.into().map(CompressionMode::Compress);
221 self
222 }
223
224 pub fn precompressed(mut self, compression: Compression) -> Self {
246 self.compression = Some(CompressionMode::Precompressed(compression));
247 self
248 }
249
250 pub fn expiration_policy(mut self, expiration_policy: ExpirationPolicy) -> Self {
254 self.metadata.expiration_policy = expiration_policy;
255 self
256 }
257
258 pub fn content_type(mut self, content_type: impl Into<Cow<'static, str>>) -> Self {
263 self.metadata.content_type = content_type.into();
264 self
265 }
266
267 pub fn origin(mut self, origin: impl Into<String>) -> Self {
285 self.metadata.origin = Some(origin.into());
286 self
287 }
288
289 pub fn filename(mut self, filename: impl Into<String>) -> Self {
295 self.metadata.filename = Some(filename.into());
296 self
297 }
298
299 pub fn set_metadata(mut self, metadata: impl Into<BTreeMap<String, String>>) -> Self {
309 self.metadata.custom = metadata.into();
310 self
311 }
312
313 pub fn append_metadata(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
321 self.metadata.custom.insert(key.into(), value.into());
322 self
323 }
324}
325
326pub(crate) struct EncodedBody {
328 pub body: Body,
329 pub length: Option<u64>,
331}
332
333impl EncodedBody {
334 pub fn sized(length: u64, body: impl Into<Body>) -> Self {
336 Self {
337 body: body.into(),
338 length: Some(length),
339 }
340 }
341
342 pub fn streamed(body: Body) -> Self {
344 Self { body, length: None }
345 }
346}
347
348pub(crate) async fn encode_body(
352 body: PutBody,
353 mode: Option<CompressionMode>,
354) -> io::Result<EncodedBody> {
355 let compression = match mode {
356 Some(CompressionMode::Compress(compression)) => Some(compression),
357 Some(CompressionMode::Precompressed(_)) | None => None,
359 };
360
361 Ok(match compression {
362 None => match body {
363 PutBody::Buffer(bytes) => EncodedBody::sized(bytes.len() as u64, bytes),
364 PutBody::Stream(stream) => EncodedBody::streamed(Body::wrap_stream(stream)),
365 PutBody::File(file) => encode_file(file).await?,
366 PutBody::Path(path) => encode_file(File::open(path).await?).await?,
367 },
368 Some(compression) => EncodedBody::streamed(match body {
369 PutBody::Buffer(bytes) => compress(Cursor::new(bytes), compression),
370 PutBody::Stream(stream) => compress(StreamReader::new(stream), compression),
371 PutBody::File(file) => compress(BufReader::new(file), compression),
372 PutBody::Path(path) => compress(BufReader::new(File::open(path).await?), compression),
373 }),
374 })
375}
376
377async fn encode_file(mut file: File) -> io::Result<EncodedBody> {
385 let metadata = file.metadata().await?;
386 if metadata.is_file() && metadata.len() > 0 {
387 let position = file.stream_position().await?;
388 let length = metadata.len().saturating_sub(position);
389 let stream = ReaderStream::new(file.take(length));
390 Ok(EncodedBody::sized(length, Body::wrap_stream(stream)))
391 } else {
392 let stream = ReaderStream::new(file);
394 Ok(EncodedBody::streamed(Body::wrap_stream(stream)))
395 }
396}
397
398fn compress(reader: impl AsyncBufRead + Send + 'static, compression: Compression) -> Body {
399 match compression {
400 Compression::Zstd => Body::wrap_stream(ReaderStream::new(ZstdEncoder::new(reader))),
401 Compression::Gzip => Body::wrap_stream(ReaderStream::new(GzipEncoder::new(reader))),
402 }
403}
404
405impl PutBuilder {
406 pub async fn send(self) -> crate::Result<PutResponse> {
408 let span = OperationSpan::storage("objectstore.put", &self.session, self.key.as_deref());
409 span.instrument(self.execute()).await
410 }
411
412 async fn execute(mut self) -> crate::Result<PutResponse> {
413 metadata::normalize_keys(&mut self.metadata)?;
414
415 let method = match self.key {
416 Some(_) => reqwest::Method::PUT,
417 None => reqwest::Method::POST,
418 };
419
420 self.metadata.compression = self.compression.map(CompressionMode::compression);
421 record_field(
422 "objectstore.compression",
423 self.metadata
424 .compression
425 .as_ref()
426 .map_or("none", Compression::as_str),
427 );
428
429 let headers = self.metadata.to_headers("")?;
430 let prepared = payload::prepare(
431 self.body,
432 self.compression,
433 self.resumable_threshold_bytes,
434 self.session.client.spool_dir().map(PathBuf::from),
435 )
436 .await?;
437 if let payload::PreparedBody::Replayable(body) = &prepared {
438 let create = CreateResumableUploadBuilder::new(
439 self.session.clone(),
440 body.len(),
441 self.key.clone(),
442 self.metadata,
443 );
444 match create.send().await {
445 Ok(Some(upload)) => {
446 let response = automatic::upload(body, upload).await?;
447 record_field("objectstore.key", response.key.as_str());
448 return Ok(response);
449 }
450 Ok(None) => {}
451 Err(_e) => {
452 debug!(error = %_e, "Resumable upload creation failed; falling back to direct upload");
453 }
454 }
455 }
456
457 let EncodedBody { body, length } = prepared.into_reqwest_body(self.compression).await?;
458 let mut builder = self
459 .session
460 .request(method, self.key.as_deref().unwrap_or_default())?;
461 builder = builder.headers(headers);
462 if let Some(length) = length {
463 builder = builder.header(CONTENT_LENGTH, length);
464 }
465 let response = builder.body(body).send().await?;
466 let response: PutResponse = response.error_for_status_and_drain().await?.json().await?;
467 record_field("objectstore.key", response.key.as_str());
468 Ok(response)
469 }
470}
471
472#[cfg(test)]
473mod tests {
474 use std::io::Read as _;
475
476 use futures_util::stream;
477 use http_body_util::BodyExt as _;
478
479 use super::*;
480
481 fn zstd_compress(data: &[u8]) -> Vec<u8> {
482 zstd::encode_all(Cursor::new(data), 0).expect("zstd encoding to succeed")
483 }
484
485 fn stream_body(chunks: Vec<&'static [u8]>) -> PutBody {
486 let chunks = chunks.into_iter().map(|c| Ok(Bytes::from_static(c)));
487 PutBody::Stream(stream::iter(chunks).boxed())
488 }
489
490 async fn collect(encoded: EncodedBody) -> Vec<u8> {
491 encoded
492 .body
493 .collect()
494 .await
495 .expect("body to be readable")
496 .to_bytes()
497 .to_vec()
498 }
499
500 #[tokio::test]
501 async fn compress_buffer_compresses() {
502 let body = PutBody::Buffer(Bytes::from_static(b"hello world"));
503 let mode = Some(CompressionMode::Compress(Compression::Zstd));
504
505 let encoded = collect(encode_body(body, mode).await.unwrap()).await;
506 assert_eq!(encoded, zstd_compress(b"hello world"));
507 }
508
509 #[tokio::test]
510 async fn compress_stream_compresses() {
511 let body = stream_body(vec![b"hello ", b"world"]);
512 let mode = Some(CompressionMode::Compress(Compression::Zstd));
513
514 let encoded = collect(encode_body(body, mode).await.unwrap()).await;
515 assert_eq!(
516 zstd::decode_all(Cursor::new(encoded)).unwrap(),
517 b"hello world"
518 );
519 }
520
521 #[tokio::test]
522 async fn gzip_body_kinds() {
523 let dir = tempfile::tempdir().unwrap();
524 let path = dir.path().join("payload");
525 std::fs::write(&path, b"hello world").unwrap();
526
527 let bodies = [
528 PutBody::Buffer(Bytes::from_static(b"hello world")),
529 stream_body(vec![b"hello ", b"world"]),
530 PutBody::File(File::open(&path).await.unwrap()),
531 PutBody::Path(path.clone()),
532 ];
533
534 for body in bodies {
535 let mode = Some(CompressionMode::Compress(Compression::Gzip));
536 let encoded = collect(encode_body(body, mode).await.unwrap()).await;
537
538 let mut decoded = Vec::new();
539 flate2::read::GzDecoder::new(&encoded[..])
540 .read_to_end(&mut decoded)
541 .unwrap();
542 assert_eq!(decoded, b"hello world");
543 }
544 }
545
546 #[tokio::test]
547 async fn precompressed_buffer_is_forwarded_verbatim() {
548 let compressed = zstd_compress(b"hello world");
549 let body = PutBody::Buffer(Bytes::from(compressed.clone()));
550 let mode = Some(CompressionMode::Precompressed(Compression::Zstd));
551
552 let encoded = collect(encode_body(body, mode).await.unwrap()).await;
553 assert_eq!(encoded, compressed);
554 }
555
556 #[tokio::test]
557 async fn precompressed_stream_is_forwarded_verbatim() {
558 let body = stream_body(vec![b"\x28\xb5\x2f\xfd", b"trailing"]);
559 let mode = Some(CompressionMode::Precompressed(Compression::Zstd));
560
561 let encoded = collect(encode_body(body, mode).await.unwrap()).await;
562 assert_eq!(encoded, b"\x28\xb5\x2f\xfdtrailing");
563 }
564
565 #[tokio::test]
566 async fn without_compression_is_forwarded_verbatim() {
567 let body = PutBody::Buffer(Bytes::from_static(b"hello world"));
568
569 let encoded = collect(encode_body(body, None).await.unwrap()).await;
570 assert_eq!(encoded, b"hello world");
571 }
572}