Skip to main content

objectstore_client/
put.rs

1use std::fmt;
2use std::io::{self, Cursor};
3use std::path::PathBuf;
4use std::{borrow::Cow, collections::BTreeMap};
5
6use async_compression::tokio::bufread::{GzipEncoder, ZstdEncoder};
7use bytes::Bytes;
8use futures_util::StreamExt;
9use objectstore_types::metadata::Metadata;
10use reqwest::Body;
11use reqwest::header::CONTENT_LENGTH;
12use serde::Deserialize;
13use tokio::fs::File;
14use tokio::io::{AsyncBufRead, AsyncRead, AsyncReadExt as _, AsyncSeekExt as _, BufReader};
15use tokio_util::io::{ReaderStream, StreamReader};
16
17pub use objectstore_types::metadata::{Compression, ExpirationPolicy};
18
19use crate::instrument::{OperationSpan, debug, record_field};
20use crate::metadata;
21use crate::response::ResponseExt as _;
22use crate::resumable::CreateResumableUploadBuilder;
23use crate::{ClientStream, ObjectKey, Session};
24
25mod automatic;
26mod payload;
27
28/// The response returned from the service after uploading an object.
29#[derive(Debug, Deserialize)]
30pub struct PutResponse {
31    /// The key of the object, as stored.
32    pub key: ObjectKey,
33}
34
35pub(crate) enum PutBody {
36    Buffer(Bytes),
37    Stream(ClientStream),
38    File(File),
39    Path(PathBuf),
40}
41
42impl fmt::Debug for PutBody {
43    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
44        f.debug_tuple("PutBody").finish_non_exhaustive()
45    }
46}
47
48/// Declares how a payload relates to the compression recorded in its metadata.
49///
50/// Both modes record the same [`Compression`] on the object; they differ only in who performs
51/// the compression.
52#[derive(Debug, Clone, Copy, PartialEq, Eq)]
53pub(crate) enum CompressionMode {
54    /// The client compresses the payload with this algorithm before uploading it.
55    Compress(Compression),
56    /// The payload is already compressed with this algorithm and is uploaded verbatim.
57    Precompressed(Compression),
58}
59
60impl CompressionMode {
61    /// Returns the compression algorithm applied to the payload.
62    pub fn compression(self) -> Compression {
63        match self {
64            Self::Compress(compression) | Self::Precompressed(compression) => compression,
65        }
66    }
67}
68
69impl Session {
70    fn put_body(&self, body: PutBody) -> PutBuilder {
71        let metadata = Metadata {
72            expiration_policy: self.scope.usecase().expiration_policy(),
73            ..Default::default()
74        };
75
76        PutBuilder {
77            session: self.clone(),
78            metadata,
79            compression: self
80                .scope
81                .usecase()
82                .compression()
83                .map(CompressionMode::Compress),
84            key: None,
85            resumable_threshold_bytes: self.scope.usecase().resumable_threshold_bytes(),
86            body,
87        }
88    }
89
90    /// Creates or replaces an object using a [`Bytes`]-like payload.
91    pub fn put(&self, body: impl Into<Bytes>) -> PutBuilder {
92        self.put_body(PutBody::Buffer(body.into()))
93    }
94
95    /// Creates or replaces an object using a streaming payload.
96    ///
97    /// Streams use direct uploads, regardless of the resumable threshold.
98    pub fn put_stream(&self, body: ClientStream) -> PutBuilder {
99        self.put_body(PutBody::Stream(body))
100    }
101
102    /// Creates or replaces an object using an [`AsyncRead`] payload.
103    ///
104    /// Readers use direct uploads. Use [`Self::put_path`] for automatic resumable file uploads.
105    pub fn put_read<R>(&self, body: R) -> PutBuilder
106    where
107        R: AsyncRead + Send + Sync + 'static,
108    {
109        let stream = ReaderStream::new(body).boxed();
110        self.put_body(PutBody::Stream(stream))
111    }
112
113    /// Creates or replaces an object using the contents of an opened file.
114    ///
115    /// Uploads from the current position using a direct request, regardless of the resumable threshold.
116    /// The file must stay readable and must not be modified until the upload completes.
117    ///
118    /// The file descriptor is held open from the moment this method is called until the
119    /// upload completes. When enqueueing many files via [`Session::many`], prefer
120    /// [`put_path`](Session::put_path) instead: it defers opening the file until just before
121    /// upload, keeping file descriptor usage within the active concurrency window and avoiding
122    /// OS file descriptor limit (e.g., macOS's default `ulimit -n`) exhaustion.
123    pub fn put_file(&self, file: File) -> PutBuilder {
124        self.put_body(PutBody::File(file))
125    }
126
127    /// Creates or replaces an object using the contents of the file at `path`.
128    ///
129    /// The path must stay readable and must not be modified, renamed, removed, or replaced
130    /// until the upload completes.
131    /// If the request is executed using a resumable upload, then multiple attempts may be
132    /// performed, and each attempt opens a new handle. Handles may remain open until pending
133    /// HTTP requests release their readers.
134    ///
135    /// Unlike [`put_file`](Session::put_file), this method defers opening the file until the
136    /// request is actually sent. When enqueueing many file uploads via [`Session::many`], this
137    /// ensures that file descriptors are opened only within the active concurrency window,
138    /// preventing the process from exhausting the OS file descriptor limit (e.g., macOS's
139    /// default `ulimit -n`).
140    ///
141    /// Prefer `put_path` over [`put_file`](Session::put_file) whenever you are lining up a
142    /// large number of files for upload.
143    pub fn put_path(&self, path: impl Into<PathBuf>) -> PutBuilder {
144        self.put_body(PutBody::Path(path.into()))
145    }
146}
147
148/// A [`put`](Session::put) request builder.
149///
150/// # Automatic resumable uploads
151///
152/// Large buffers and paths to regular files use resumable uploads automatically,
153/// recovering from interrupted requests. Open files, streams, readers, and uploads queued through
154/// [`Session::many`] do not use resumables. No feature flag is required.
155///
156/// Configure this with [`Self::resumable_threshold_bytes`] and [`crate::ClientBuilder::spool_dir`].
157#[derive(Debug)]
158pub struct PutBuilder {
159    pub(crate) session: Session,
160    pub(crate) metadata: Metadata,
161    pub(crate) compression: Option<CompressionMode>,
162    pub(crate) key: Option<ObjectKey>,
163    pub(crate) body: PutBody,
164    pub(crate) resumable_threshold_bytes: Option<u64>,
165}
166
167impl PutBuilder {
168    /// Sets the minimum remaining input size for automatic resumable uploads.
169    ///
170    /// Measures remaining bytes before client compression. Inherits the
171    /// [`Usecase` threshold](crate::Usecase::with_resumable_threshold_bytes) unless overridden.
172    /// Pass [`None`] to disable or zero to include every supported payload.
173    /// See [`PutBuilder`] for eligibility.
174    ///
175    /// ```no_run
176    /// # async fn example(session: objectstore_client::Session) -> objectstore_client::Result<()> {
177    /// session.put("payload").resumable_threshold_bytes(None).send().await?;
178    /// # Ok(())
179    /// # }
180    /// ```
181    pub fn resumable_threshold_bytes(mut self, threshold: impl Into<Option<u64>>) -> Self {
182        self.resumable_threshold_bytes = threshold.into();
183        self
184    }
185
186    /// Sets an explicit object key.
187    ///
188    /// If a key is specified, the object will be stored under that key. Otherwise, the Objectstore
189    /// server will automatically assign a random key, which is then returned from this request.
190    pub fn key(mut self, key: impl Into<ObjectKey>) -> Self {
191        self.key = Some(key.into()).filter(|k| !k.is_empty());
192        self
193    }
194
195    /// Sets an explicit compression algorithm to be used for this payload.
196    ///
197    /// The client compresses the payload and records the algorithm in the
198    /// object's metadata. [`None`] should be used if no compression should be performed by the
199    /// client, either because the payload is uncompressible (such as a media format), or if the
200    /// compression should not be recorded for this object.
201    ///
202    /// If the payload is already compressed and the algorithm should still be recorded, use
203    /// [`precompressed`](Self::precompressed) instead.
204    ///
205    /// By default, the compression algorithm set on this Session's Usecase is used (see
206    /// [`with_compression`](crate::Usecase::with_compression)).
207    ///
208    /// # Example
209    ///
210    /// ```no_run
211    /// # async fn example(session: objectstore_client::Session, media: Vec<u8>) {
212    /// session.put(media)
213    ///     .compress(None) // uncompressible payload
214    ///     .send()
215    ///     .await
216    ///     .unwrap();
217    /// # }
218    /// ```
219    pub fn compress(mut self, compression: impl Into<Option<Compression>>) -> Self {
220        self.compression = compression.into().map(CompressionMode::Compress);
221        self
222    }
223
224    /// Declares that the payload is already compressed with the given algorithm.
225    ///
226    /// The payload is uploaded verbatim, and the algorithm is recorded in the object's metadata
227    /// so that downloads decompress it transparently. Use this to hand pre-compressed data to
228    /// the client without paying for another compression pass.
229    ///
230    /// This overrides the compression algorithm set on this Session's Usecase. To have the
231    /// client perform the compression instead, use [`compress`](Self::compress).
232    ///
233    /// # Example
234    ///
235    /// ```no_run
236    /// # use objectstore_client::Compression;
237    /// # async fn example(session: objectstore_client::Session, zstd_data: Vec<u8>) {
238    /// session.put(zstd_data)
239    ///     .precompressed(Compression::Zstd)
240    ///     .send()
241    ///     .await
242    ///     .unwrap();
243    /// # }
244    /// ```
245    pub fn precompressed(mut self, compression: Compression) -> Self {
246        self.compression = Some(CompressionMode::Precompressed(compression));
247        self
248    }
249
250    /// Sets the expiration policy of the object to be uploaded.
251    ///
252    /// By default, the expiration policy set on this Session's Usecase is used.
253    pub fn expiration_policy(mut self, expiration_policy: ExpirationPolicy) -> Self {
254        self.metadata.expiration_policy = expiration_policy;
255        self
256    }
257
258    /// Sets the content type of the object to be uploaded.
259    ///
260    /// You can use the utility function [`crate::utils::guess_mime_type`] to attempt to guess a
261    /// `content_type` based on magic bytes.
262    pub fn content_type(mut self, content_type: impl Into<Cow<'static, str>>) -> Self {
263        self.metadata.content_type = content_type.into();
264        self
265    }
266
267    /// Sets the origin of the object, typically the IP address of the original source.
268    ///
269    /// This is an optional but encouraged field that tracks where the payload was
270    /// originally obtained from. For example, the IP address of the Sentry SDK or CLI
271    /// that uploaded the data.
272    ///
273    /// # Example
274    ///
275    /// ```no_run
276    /// # async fn example(session: objectstore_client::Session) {
277    /// session.put("data")
278    ///     .origin("203.0.113.42")
279    ///     .send()
280    ///     .await
281    ///     .unwrap();
282    /// # }
283    /// ```
284    pub fn origin(mut self, origin: impl Into<String>) -> Self {
285        self.metadata.origin = Some(origin.into());
286        self
287    }
288
289    /// Sets the filename of the object.
290    ///
291    /// When present, the server will include a `Content-Disposition: attachment; filename="<filename>"`
292    /// header in GET responses, prompting browsers and download tools to save the file under
293    /// this name.
294    pub fn filename(mut self, filename: impl Into<String>) -> Self {
295        self.metadata.filename = Some(filename.into());
296        self
297    }
298
299    /// This sets the custom metadata to the provided map.
300    ///
301    /// It will clear any previously set metadata.
302    ///
303    /// Keys are case-insensitive: ASCII letters are lowercased. A key must be non-empty and
304    /// consist only of `a-z`, `0-9`, and the symbols `_ - | ~ % $ # + . :`. Keys are checked when
305    /// the request is sent, not here, so chaining never fails. [`send`](PutBuilder::send) returns
306    /// [`Error::Metadata`](crate::Error::Metadata) naming the first invalid key and sends no
307    /// request.
308    pub fn set_metadata(mut self, metadata: impl Into<BTreeMap<String, String>>) -> Self {
309        self.metadata.custom = metadata.into();
310        self
311    }
312
313    /// Appends the `key`/`value` to the custom metadata of this object.
314    ///
315    /// Keys are case-insensitive: ASCII letters are lowercased. A key must be non-empty and
316    /// consist only of `a-z`, `0-9`, and the symbols `_ - | ~ % $ # + . :`. Keys are checked when
317    /// the request is sent, not here, so chaining never fails. [`send`](PutBuilder::send) returns
318    /// [`Error::Metadata`](crate::Error::Metadata) naming the first invalid key and sends no
319    /// request.
320    pub fn append_metadata(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
321        self.metadata.custom.insert(key.into(), value.into());
322        self
323    }
324}
325
326/// A request body along with its exact length, if known before sending.
327pub(crate) struct EncodedBody {
328    pub body: Body,
329    /// The number of bytes `body` yields, sent as `Content-Length` when known.
330    pub length: Option<u64>,
331}
332
333impl EncodedBody {
334    /// Creates a body whose length is known up front.
335    pub fn sized(length: u64, body: impl Into<Body>) -> Self {
336        Self {
337            body: body.into(),
338            length: Some(length),
339        }
340    }
341
342    /// Creates a body whose length is only known once it has been streamed.
343    pub fn streamed(body: Body) -> Self {
344        Self { body, length: None }
345    }
346}
347
348/// Turns the body into a request body, compressing it if the mode asks for it.
349///
350/// Payloads declared as [`CompressionMode::Precompressed`] are forwarded verbatim.
351pub(crate) async fn encode_body(
352    body: PutBody,
353    mode: Option<CompressionMode>,
354) -> io::Result<EncodedBody> {
355    let compression = match mode {
356        Some(CompressionMode::Compress(compression)) => Some(compression),
357        // The payload already carries the encoding, so nothing is left to do here.
358        Some(CompressionMode::Precompressed(_)) | None => None,
359    };
360
361    Ok(match compression {
362        None => match body {
363            PutBody::Buffer(bytes) => EncodedBody::sized(bytes.len() as u64, bytes),
364            PutBody::Stream(stream) => EncodedBody::streamed(Body::wrap_stream(stream)),
365            PutBody::File(file) => encode_file(file).await?,
366            PutBody::Path(path) => encode_file(File::open(path).await?).await?,
367        },
368        Some(compression) => EncodedBody::streamed(match body {
369            PutBody::Buffer(bytes) => compress(Cursor::new(bytes), compression),
370            PutBody::Stream(stream) => compress(StreamReader::new(stream), compression),
371            PutBody::File(file) => compress(BufReader::new(file), compression),
372            PutBody::Path(path) => compress(BufReader::new(File::open(path).await?), compression),
373        }),
374    })
375}
376
377/// Streams the remainder of a file from its current position.
378///
379/// For regular files, the length is taken from the file's metadata and the stream is capped to
380/// it, so that a file growing during the upload cannot exceed the declared length.
381///
382/// A reported size of zero is not trusted: virtual files such as those in `/proc` report zero
383/// bytes but still have content. These files, and empty ones, are streamed without a length.
384async fn encode_file(mut file: File) -> io::Result<EncodedBody> {
385    let metadata = file.metadata().await?;
386    if metadata.is_file() && metadata.len() > 0 {
387        let position = file.stream_position().await?;
388        let length = metadata.len().saturating_sub(position);
389        let stream = ReaderStream::new(file.take(length));
390        Ok(EncodedBody::sized(length, Body::wrap_stream(stream)))
391    } else {
392        // Pipes, special files, and virtual files do not report a meaningful size.
393        let stream = ReaderStream::new(file);
394        Ok(EncodedBody::streamed(Body::wrap_stream(stream)))
395    }
396}
397
398fn compress(reader: impl AsyncBufRead + Send + 'static, compression: Compression) -> Body {
399    match compression {
400        Compression::Zstd => Body::wrap_stream(ReaderStream::new(ZstdEncoder::new(reader))),
401        Compression::Gzip => Body::wrap_stream(ReaderStream::new(GzipEncoder::new(reader))),
402    }
403}
404
405impl PutBuilder {
406    /// Sends the built put request to the upstream service.
407    pub async fn send(self) -> crate::Result<PutResponse> {
408        let span = OperationSpan::storage("objectstore.put", &self.session, self.key.as_deref());
409        span.instrument(self.execute()).await
410    }
411
412    async fn execute(mut self) -> crate::Result<PutResponse> {
413        metadata::normalize_keys(&mut self.metadata)?;
414
415        let method = match self.key {
416            Some(_) => reqwest::Method::PUT,
417            None => reqwest::Method::POST,
418        };
419
420        self.metadata.compression = self.compression.map(CompressionMode::compression);
421        record_field(
422            "objectstore.compression",
423            self.metadata
424                .compression
425                .as_ref()
426                .map_or("none", Compression::as_str),
427        );
428
429        let headers = self.metadata.to_headers("")?;
430        let prepared = payload::prepare(
431            self.body,
432            self.compression,
433            self.resumable_threshold_bytes,
434            self.session.client.spool_dir().map(PathBuf::from),
435        )
436        .await?;
437        if let payload::PreparedBody::Replayable(body) = &prepared {
438            let create = CreateResumableUploadBuilder::new(
439                self.session.clone(),
440                body.len(),
441                self.key.clone(),
442                self.metadata,
443            );
444            match create.send().await {
445                Ok(Some(upload)) => {
446                    let response = automatic::upload(body, upload).await?;
447                    record_field("objectstore.key", response.key.as_str());
448                    return Ok(response);
449                }
450                Ok(None) => {}
451                Err(_e) => {
452                    debug!(error = %_e, "Resumable upload creation failed; falling back to direct upload");
453                }
454            }
455        }
456
457        let EncodedBody { body, length } = prepared.into_reqwest_body(self.compression).await?;
458        let mut builder = self
459            .session
460            .request(method, self.key.as_deref().unwrap_or_default())?;
461        builder = builder.headers(headers);
462        if let Some(length) = length {
463            builder = builder.header(CONTENT_LENGTH, length);
464        }
465        let response = builder.body(body).send().await?;
466        let response: PutResponse = response.error_for_status_and_drain().await?.json().await?;
467        record_field("objectstore.key", response.key.as_str());
468        Ok(response)
469    }
470}
471
472#[cfg(test)]
473mod tests {
474    use std::io::Read as _;
475
476    use futures_util::stream;
477    use http_body_util::BodyExt as _;
478
479    use super::*;
480
481    fn zstd_compress(data: &[u8]) -> Vec<u8> {
482        zstd::encode_all(Cursor::new(data), 0).expect("zstd encoding to succeed")
483    }
484
485    fn stream_body(chunks: Vec<&'static [u8]>) -> PutBody {
486        let chunks = chunks.into_iter().map(|c| Ok(Bytes::from_static(c)));
487        PutBody::Stream(stream::iter(chunks).boxed())
488    }
489
490    async fn collect(encoded: EncodedBody) -> Vec<u8> {
491        encoded
492            .body
493            .collect()
494            .await
495            .expect("body to be readable")
496            .to_bytes()
497            .to_vec()
498    }
499
500    #[tokio::test]
501    async fn compress_buffer_compresses() {
502        let body = PutBody::Buffer(Bytes::from_static(b"hello world"));
503        let mode = Some(CompressionMode::Compress(Compression::Zstd));
504
505        let encoded = collect(encode_body(body, mode).await.unwrap()).await;
506        assert_eq!(encoded, zstd_compress(b"hello world"));
507    }
508
509    #[tokio::test]
510    async fn compress_stream_compresses() {
511        let body = stream_body(vec![b"hello ", b"world"]);
512        let mode = Some(CompressionMode::Compress(Compression::Zstd));
513
514        let encoded = collect(encode_body(body, mode).await.unwrap()).await;
515        assert_eq!(
516            zstd::decode_all(Cursor::new(encoded)).unwrap(),
517            b"hello world"
518        );
519    }
520
521    #[tokio::test]
522    async fn gzip_body_kinds() {
523        let dir = tempfile::tempdir().unwrap();
524        let path = dir.path().join("payload");
525        std::fs::write(&path, b"hello world").unwrap();
526
527        let bodies = [
528            PutBody::Buffer(Bytes::from_static(b"hello world")),
529            stream_body(vec![b"hello ", b"world"]),
530            PutBody::File(File::open(&path).await.unwrap()),
531            PutBody::Path(path.clone()),
532        ];
533
534        for body in bodies {
535            let mode = Some(CompressionMode::Compress(Compression::Gzip));
536            let encoded = collect(encode_body(body, mode).await.unwrap()).await;
537
538            let mut decoded = Vec::new();
539            flate2::read::GzDecoder::new(&encoded[..])
540                .read_to_end(&mut decoded)
541                .unwrap();
542            assert_eq!(decoded, b"hello world");
543        }
544    }
545
546    #[tokio::test]
547    async fn precompressed_buffer_is_forwarded_verbatim() {
548        let compressed = zstd_compress(b"hello world");
549        let body = PutBody::Buffer(Bytes::from(compressed.clone()));
550        let mode = Some(CompressionMode::Precompressed(Compression::Zstd));
551
552        let encoded = collect(encode_body(body, mode).await.unwrap()).await;
553        assert_eq!(encoded, compressed);
554    }
555
556    #[tokio::test]
557    async fn precompressed_stream_is_forwarded_verbatim() {
558        let body = stream_body(vec![b"\x28\xb5\x2f\xfd", b"trailing"]);
559        let mode = Some(CompressionMode::Precompressed(Compression::Zstd));
560
561        let encoded = collect(encode_body(body, mode).await.unwrap()).await;
562        assert_eq!(encoded, b"\x28\xb5\x2f\xfdtrailing");
563    }
564
565    #[tokio::test]
566    async fn without_compression_is_forwarded_verbatim() {
567        let body = PutBody::Buffer(Bytes::from_static(b"hello world"));
568
569        let encoded = collect(encode_body(body, None).await.unwrap()).await;
570        assert_eq!(encoded, b"hello world");
571    }
572}