objectstore_client/
put.rs

1use std::fmt;
2use std::io::{self, Cursor};
3use std::path::PathBuf;
4use std::{borrow::Cow, collections::BTreeMap};
5
6use async_compression::tokio::bufread::ZstdEncoder;
7use bytes::Bytes;
8use futures_util::StreamExt;
9use objectstore_types::metadata::Metadata;
10use reqwest::Body;
11use serde::Deserialize;
12use tokio::fs::File;
13use tokio::io::{AsyncRead, BufReader};
14use tokio_util::io::{ReaderStream, StreamReader};
15
16pub use objectstore_types::metadata::{Compression, ExpirationPolicy};
17
18use crate::response::ResponseExt as _;
19use crate::{ClientStream, ObjectKey, Session};
20
21/// The response returned from the service after uploading an object.
22#[derive(Debug, Deserialize)]
23pub struct PutResponse {
24    /// The key of the object, as stored.
25    pub key: ObjectKey,
26}
27
28pub(crate) enum PutBody {
29    Buffer(Bytes),
30    Stream(ClientStream),
31    File(File),
32    Path(PathBuf),
33}
34
35impl fmt::Debug for PutBody {
36    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
37        f.debug_tuple("PutBody").finish_non_exhaustive()
38    }
39}
40
41/// Declares how a payload relates to the compression recorded in its metadata.
42///
43/// Both modes record the same [`Compression`] on the object; they differ only in who performs
44/// the compression.
45#[derive(Debug, Clone, Copy, PartialEq, Eq)]
46pub(crate) enum CompressionMode {
47    /// The client compresses the payload with this algorithm before uploading it.
48    Compress(Compression),
49    /// The payload is already compressed with this algorithm and is uploaded verbatim.
50    Precompressed(Compression),
51}
52
53impl CompressionMode {
54    /// Returns the compression algorithm applied to the payload.
55    pub fn compression(self) -> Compression {
56        match self {
57            Self::Compress(compression) | Self::Precompressed(compression) => compression,
58        }
59    }
60}
61
62impl Session {
63    fn put_body(&self, body: PutBody) -> PutBuilder {
64        let metadata = Metadata {
65            expiration_policy: self.scope.usecase().expiration_policy(),
66            ..Default::default()
67        };
68
69        PutBuilder {
70            session: self.clone(),
71            metadata,
72            compression: self
73                .scope
74                .usecase()
75                .compression()
76                .map(CompressionMode::Compress),
77            key: None,
78            body,
79        }
80    }
81
82    /// Creates or replaces an object using a [`Bytes`]-like payload.
83    pub fn put(&self, body: impl Into<Bytes>) -> PutBuilder {
84        self.put_body(PutBody::Buffer(body.into()))
85    }
86
87    /// Creates or replaces an object using a streaming payload.
88    pub fn put_stream(&self, body: ClientStream) -> PutBuilder {
89        self.put_body(PutBody::Stream(body))
90    }
91
92    /// Creates or replaces an object using an [`AsyncRead`] payload.
93    pub fn put_read<R>(&self, body: R) -> PutBuilder
94    where
95        R: AsyncRead + Send + Sync + 'static,
96    {
97        let stream = ReaderStream::new(body).boxed();
98        self.put_body(PutBody::Stream(stream))
99    }
100
101    /// Creates or replaces an object using the contents of an opened file.
102    ///
103    /// The file descriptor is held open from the moment this method is called until the
104    /// upload completes. When enqueueing many files via [`Session::many`], prefer
105    /// [`put_path`](Session::put_path) instead: it defers opening the file until just before
106    /// upload, keeping file descriptor usage within the active concurrency window and avoiding
107    /// OS file descriptor limit (e.g., macOS's default `ulimit -n`) exhaustion.
108    pub fn put_file(&self, file: File) -> PutBuilder {
109        self.put_body(PutBody::File(file))
110    }
111
112    /// Creates or replaces an object using the contents of the file at `path`.
113    ///
114    /// Unlike [`put_file`](Session::put_file), this method defers opening the file until the
115    /// request is actually sent. When enqueueing many file uploads via [`Session::many`], this
116    /// ensures that file descriptors are opened only within the active concurrency window,
117    /// preventing the process from exhausting the OS file descriptor limit (e.g., macOS's
118    /// default `ulimit -n`).
119    ///
120    /// Prefer `put_path` over [`put_file`](Session::put_file) whenever you are lining up a
121    /// large number of files for upload.
122    pub fn put_path(&self, path: impl Into<PathBuf>) -> PutBuilder {
123        self.put_body(PutBody::Path(path.into()))
124    }
125}
126
127/// A [`put`](Session::put) request builder.
128#[derive(Debug)]
129pub struct PutBuilder {
130    pub(crate) session: Session,
131    pub(crate) metadata: Metadata,
132    pub(crate) compression: Option<CompressionMode>,
133    pub(crate) key: Option<ObjectKey>,
134    pub(crate) body: PutBody,
135}
136
137impl PutBuilder {
138    /// Sets an explicit object key.
139    ///
140    /// If a key is specified, the object will be stored under that key. Otherwise, the Objectstore
141    /// server will automatically assign a random key, which is then returned from this request.
142    pub fn key(mut self, key: impl Into<ObjectKey>) -> Self {
143        self.key = Some(key.into()).filter(|k| !k.is_empty());
144        self
145    }
146
147    /// Sets an explicit compression algorithm to be used for this payload.
148    ///
149    /// The client compresses the payload while uploading it and records the algorithm in the
150    /// object's metadata. [`None`] should be used if no compression should be performed by the
151    /// client, either because the payload is uncompressible (such as a media format), or if the
152    /// compression should not be recorded for this object.
153    ///
154    /// If the payload is already compressed and the algorithm should still be recorded, use
155    /// [`precompressed`](Self::precompressed) instead.
156    ///
157    /// By default, the compression algorithm set on this Session's Usecase is used (see
158    /// [`with_compression`](crate::Usecase::with_compression)).
159    ///
160    /// # Example
161    ///
162    /// ```no_run
163    /// # async fn example(session: objectstore_client::Session, media: Vec<u8>) {
164    /// session.put(media)
165    ///     .compress(None) // uncompressible payload
166    ///     .send()
167    ///     .await
168    ///     .unwrap();
169    /// # }
170    /// ```
171    pub fn compress(mut self, compression: impl Into<Option<Compression>>) -> Self {
172        self.compression = compression.into().map(CompressionMode::Compress);
173        self
174    }
175
176    /// Deprecated in favor of [`compress`](Self::compress).
177    #[deprecated(since = "0.3.0", note = "renamed to `compress`")]
178    pub fn compression(self, compression: impl Into<Option<Compression>>) -> Self {
179        self.compress(compression)
180    }
181
182    /// Declares that the payload is already compressed with the given algorithm.
183    ///
184    /// The payload is uploaded verbatim, and the algorithm is recorded in the object's metadata
185    /// so that downloads decompress it transparently. Use this to hand pre-compressed data to
186    /// the client without paying for another compression pass.
187    ///
188    /// This overrides the compression algorithm set on this Session's Usecase. To have the
189    /// client perform the compression instead, use [`compress`](Self::compress).
190    ///
191    /// # Example
192    ///
193    /// ```no_run
194    /// # use objectstore_client::Compression;
195    /// # async fn example(session: objectstore_client::Session, zstd_data: Vec<u8>) {
196    /// session.put(zstd_data)
197    ///     .precompressed(Compression::Zstd)
198    ///     .send()
199    ///     .await
200    ///     .unwrap();
201    /// # }
202    /// ```
203    pub fn precompressed(mut self, compression: Compression) -> Self {
204        self.compression = Some(CompressionMode::Precompressed(compression));
205        self
206    }
207
208    /// Sets the expiration policy of the object to be uploaded.
209    ///
210    /// By default, the expiration policy set on this Session's Usecase is used.
211    pub fn expiration_policy(mut self, expiration_policy: ExpirationPolicy) -> Self {
212        self.metadata.expiration_policy = expiration_policy;
213        self
214    }
215
216    /// Sets the content type of the object to be uploaded.
217    ///
218    /// You can use the utility function [`crate::utils::guess_mime_type`] to attempt to guess a
219    /// `content_type` based on magic bytes.
220    pub fn content_type(mut self, content_type: impl Into<Cow<'static, str>>) -> Self {
221        self.metadata.content_type = content_type.into();
222        self
223    }
224
225    /// Sets the origin of the object, typically the IP address of the original source.
226    ///
227    /// This is an optional but encouraged field that tracks where the payload was
228    /// originally obtained from. For example, the IP address of the Sentry SDK or CLI
229    /// that uploaded the data.
230    ///
231    /// # Example
232    ///
233    /// ```no_run
234    /// # async fn example(session: objectstore_client::Session) {
235    /// session.put("data")
236    ///     .origin("203.0.113.42")
237    ///     .send()
238    ///     .await
239    ///     .unwrap();
240    /// # }
241    /// ```
242    pub fn origin(mut self, origin: impl Into<String>) -> Self {
243        self.metadata.origin = Some(origin.into());
244        self
245    }
246
247    /// Sets the filename of the object.
248    ///
249    /// When present, the server will include a `Content-Disposition: attachment; filename="<filename>"`
250    /// header in GET responses, prompting browsers and download tools to save the file under
251    /// this name.
252    pub fn filename(mut self, filename: impl Into<String>) -> Self {
253        self.metadata.filename = Some(filename.into());
254        self
255    }
256
257    /// This sets the custom metadata to the provided map.
258    ///
259    /// It will clear any previously set metadata.
260    pub fn set_metadata(mut self, metadata: impl Into<BTreeMap<String, String>>) -> Self {
261        self.metadata.custom = metadata.into();
262        self
263    }
264
265    /// Appends they `key`/`value` to the custom metadata of this object.
266    pub fn append_metadata(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
267        self.metadata.custom.insert(key.into(), value.into());
268        self
269    }
270}
271
272/// Turns the body into a request body, compressing it if the mode asks for it.
273///
274/// Payloads declared as [`CompressionMode::Precompressed`] are forwarded verbatim.
275pub(crate) async fn encode_body(body: PutBody, mode: Option<CompressionMode>) -> io::Result<Body> {
276    let compression = match mode {
277        Some(CompressionMode::Compress(compression)) => Some(compression),
278        // The payload already carries the encoding, so nothing is left to do here.
279        Some(CompressionMode::Precompressed(_)) | None => None,
280    };
281
282    Ok(match (compression, body) {
283        (Some(Compression::Zstd), PutBody::Buffer(bytes)) => {
284            let cursor = Cursor::new(bytes);
285            let encoder = ZstdEncoder::new(cursor);
286            let stream = ReaderStream::new(encoder);
287            Body::wrap_stream(stream)
288        }
289        (Some(Compression::Zstd), PutBody::Stream(stream)) => {
290            let stream = StreamReader::new(stream);
291            let encoder = ZstdEncoder::new(stream);
292            let stream = ReaderStream::new(encoder);
293            Body::wrap_stream(stream)
294        }
295        (Some(Compression::Zstd), PutBody::File(file)) => {
296            let reader = BufReader::new(file);
297            let encoder = ZstdEncoder::new(reader);
298            let stream = ReaderStream::new(encoder);
299            Body::wrap_stream(stream)
300        }
301        (Some(Compression::Zstd), PutBody::Path(file)) => {
302            let file = File::open(file).await?;
303            let reader = BufReader::new(file);
304            let encoder = ZstdEncoder::new(reader);
305            let stream = ReaderStream::new(encoder);
306            Body::wrap_stream(stream)
307        }
308        (None, PutBody::Buffer(bytes)) => bytes.into(),
309        (None, PutBody::Stream(stream)) => Body::wrap_stream(stream),
310        (None, PutBody::File(file)) => {
311            let stream = ReaderStream::new(file);
312            Body::wrap_stream(stream)
313        }
314        (None, PutBody::Path(path)) => {
315            let stream = ReaderStream::new(File::open(path).await?);
316            Body::wrap_stream(stream)
317        }
318    })
319}
320
321// TODO: instead of a separate `send` method, it would be nice to just implement `IntoFuture`.
322// However, `IntoFuture` needs to define the resulting future as an associated type,
323// and "impl trait in associated type position" is not yet stable :-(
324impl PutBuilder {
325    /// Sends the built put request to the upstream service.
326    pub async fn send(mut self) -> crate::Result<PutResponse> {
327        let method = match self.key {
328            Some(_) => reqwest::Method::PUT,
329            None => reqwest::Method::POST,
330        };
331
332        let mut builder = self
333            .session
334            .request(method, self.key.as_deref().unwrap_or_default())?;
335
336        self.metadata.compression = self.compression.map(CompressionMode::compression);
337        let body = encode_body(self.body, self.compression).await?;
338
339        builder = builder.headers(self.metadata.to_headers("")?);
340
341        let response = builder.body(body).send().await?;
342        Ok(response.error_for_status_and_drain().await?.json().await?)
343    }
344}
345
346#[cfg(test)]
347mod tests {
348    use futures_util::stream;
349    use http_body_util::BodyExt as _;
350
351    use super::*;
352
353    fn zstd_compress(data: &[u8]) -> Vec<u8> {
354        zstd::encode_all(Cursor::new(data), 0).expect("zstd encoding to succeed")
355    }
356
357    fn stream_body(chunks: Vec<&'static [u8]>) -> PutBody {
358        let chunks = chunks.into_iter().map(|c| Ok(Bytes::from_static(c)));
359        PutBody::Stream(stream::iter(chunks).boxed())
360    }
361
362    async fn collect(body: Body) -> Vec<u8> {
363        body.collect()
364            .await
365            .expect("body to be readable")
366            .to_bytes()
367            .to_vec()
368    }
369
370    #[tokio::test]
371    async fn compress_buffer_compresses() {
372        let body = PutBody::Buffer(Bytes::from_static(b"hello world"));
373        let mode = Some(CompressionMode::Compress(Compression::Zstd));
374
375        let encoded = collect(encode_body(body, mode).await.unwrap()).await;
376        assert_eq!(encoded, zstd_compress(b"hello world"));
377    }
378
379    #[tokio::test]
380    async fn compress_stream_compresses() {
381        let body = stream_body(vec![b"hello ", b"world"]);
382        let mode = Some(CompressionMode::Compress(Compression::Zstd));
383
384        let encoded = collect(encode_body(body, mode).await.unwrap()).await;
385        assert_eq!(
386            zstd::decode_all(Cursor::new(encoded)).unwrap(),
387            b"hello world"
388        );
389    }
390
391    #[tokio::test]
392    async fn precompressed_buffer_is_forwarded_verbatim() {
393        let compressed = zstd_compress(b"hello world");
394        let body = PutBody::Buffer(Bytes::from(compressed.clone()));
395        let mode = Some(CompressionMode::Precompressed(Compression::Zstd));
396
397        let encoded = collect(encode_body(body, mode).await.unwrap()).await;
398        assert_eq!(encoded, compressed);
399    }
400
401    #[tokio::test]
402    async fn precompressed_stream_is_forwarded_verbatim() {
403        let body = stream_body(vec![b"\x28\xb5\x2f\xfd", b"trailing"]);
404        let mode = Some(CompressionMode::Precompressed(Compression::Zstd));
405
406        let encoded = collect(encode_body(body, mode).await.unwrap()).await;
407        assert_eq!(encoded, b"\x28\xb5\x2f\xfdtrailing");
408    }
409
410    #[tokio::test]
411    async fn without_compression_is_forwarded_verbatim() {
412        let body = PutBody::Buffer(Bytes::from_static(b"hello world"));
413
414        let encoded = collect(encode_body(body, None).await.unwrap()).await;
415        assert_eq!(encoded, b"hello world");
416    }
417}
objectstore_client/put.rs

objectstore_client/
put.rs
