Skip to main content

objectstore_client/
get.rs

1use std::{fmt, io};
2
3use async_compression::tokio::bufread::{GzipDecoder, ZstdDecoder};
4use bytes::BytesMut;
5use futures_util::{StreamExt, TryStreamExt};
6use objectstore_types::metadata::{Compression, Metadata};
7use reqwest::StatusCode;
8use tokio_util::io::{ReaderStream, StreamReader};
9
10use crate::instrument::{OperationSpan, record_field};
11use crate::response::ResponseExt as _;
12use crate::{ClientStream, ObjectKey, Session};
13
14/// The result from a successful [`get()`](Session::get) call.
15///
16/// This carries the response as a stream, plus the compression algorithm of the data.
17pub struct GetResponse {
18    /// The metadata attached to this object, including the compression algorithm used for the payload.
19    pub metadata: Metadata,
20    /// The response stream.
21    pub stream: ClientStream,
22}
23
24impl GetResponse {
25    /// Loads the object payload fully into memory.
26    pub async fn payload(self) -> crate::Result<bytes::Bytes> {
27        let bytes: BytesMut = self.stream.try_collect().await?;
28        Ok(bytes.freeze())
29    }
30
31    /// Loads the object payload fully into memory and interprets it as UTF-8 text.
32    pub async fn text(self) -> crate::Result<String> {
33        let bytes = self.payload().await?;
34        Ok(String::from_utf8(bytes.to_vec())?)
35    }
36}
37
38impl fmt::Debug for GetResponse {
39    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
40        f.debug_struct("GetResponse")
41            .field("metadata", &self.metadata)
42            .field("stream", &format_args!("[Stream]"))
43            .finish()
44    }
45}
46
47impl Session {
48    /// Retrieves the object with the given `key`.
49    pub fn get(&self, key: &str) -> GetBuilder {
50        GetBuilder {
51            session: self.clone(),
52            key: key.to_owned(),
53            decompress: true,
54            accept_encoding: vec![],
55        }
56    }
57}
58
59/// A [`get`](Session::get) request builder.
60#[derive(Debug)]
61pub struct GetBuilder {
62    pub(crate) session: Session,
63    pub(crate) key: ObjectKey,
64    pub(crate) decompress: bool,
65    pub(crate) accept_encoding: Vec<Compression>,
66}
67
68impl GetBuilder {
69    /// Indicates whether the request should automatically handle decompression of known algorithms,
70    /// or rather return the payload as it is stored, along with the compression algorithm it is stored in.
71    ///
72    /// By default, automatic decompression is enabled.
73    pub fn decompress(mut self, decompress: bool) -> Self {
74        self.decompress = decompress;
75        self
76    }
77
78    /// Specifies compression encodings the caller can handle natively.
79    ///
80    /// When the stored object's compression matches one of these, the payload
81    /// is returned still compressed and `metadata.compression` is preserved.
82    /// An empty list (the default) means the client does not accept any
83    /// compressed encoding, so automatic decompression applies as usual.
84    pub fn accept_encoding(mut self, encodings: impl IntoIterator<Item = Compression>) -> Self {
85        self.accept_encoding = encodings.into_iter().collect();
86        self
87    }
88
89    /// Sends the get request.
90    pub async fn send(self) -> crate::Result<Option<GetResponse>> {
91        let span = OperationSpan::storage("objectstore.get", &self.session, Some(&self.key));
92        span.instrument(self.execute()).await
93    }
94
95    async fn execute(self) -> crate::Result<Option<GetResponse>> {
96        let response = self
97            .session
98            .request(reqwest::Method::GET, &self.key)?
99            .send()
100            .await?;
101        if response.status() == StatusCode::NOT_FOUND {
102            record_field("objectstore.found", false);
103            response.drain_body().await;
104            return Ok(None);
105        }
106        let response = response.error_for_status_and_drain().await?;
107        record_field("objectstore.found", true);
108
109        let mut metadata = match Metadata::from_headers(response.headers(), "") {
110            Ok(metadata) => metadata,
111            Err(error) => {
112                response.drain_body().await;
113                return Err(error.into());
114            }
115        };
116
117        let stored_compression = metadata.compression;
118        let stream = response.bytes_stream().map_err(io::Error::other).boxed();
119        let stream = maybe_decompress(
120            stream,
121            &mut metadata,
122            self.decompress,
123            &self.accept_encoding,
124        );
125        record_field(
126            "objectstore.compression",
127            stored_compression
128                .as_ref()
129                .map_or("none", Compression::as_str),
130        );
131        record_field(
132            "objectstore.decompressed",
133            metadata.compression != stored_compression,
134        );
135
136        Ok(Some(GetResponse { metadata, stream }))
137    }
138}
139
140/// Wraps a stream in a decompression layer.
141///
142/// Decompresses if the metadata indicates compression, `decompress` is `true`,
143/// and the stored encoding is not listed in `accept_encoding`. When the stored encoding
144/// is in `accept_encoding`, the payload is returned compressed and `metadata.compression`
145/// is preserved. Clears `metadata.compression` when decompression is applied.
146pub(crate) fn maybe_decompress(
147    stream: ClientStream,
148    metadata: &mut Metadata,
149    decompress: bool,
150    accept_encoding: &[Compression],
151) -> ClientStream {
152    let encoding_accepted = metadata
153        .compression
154        .is_some_and(|c| accept_encoding.contains(&c));
155    match (metadata.compression, decompress && !encoding_accepted) {
156        (Some(Compression::Zstd), true) => {
157            metadata.compression = None;
158            let mut decoder = ZstdDecoder::new(StreamReader::new(stream));
159            // Zstd streams may contain concatenated frames (RFC 8878 §3.1).
160            decoder.multiple_members(true);
161            ReaderStream::new(decoder).boxed()
162        }
163        (Some(Compression::Gzip), true) => {
164            metadata.compression = None;
165            let mut decoder = GzipDecoder::new(StreamReader::new(stream));
166            // Gzip streams may contain concatenated members (RFC 1952 §2.2).
167            decoder.multiple_members(true);
168            ReaderStream::new(decoder).boxed()
169        }
170        (Some(_), false) | (None, _) => stream,
171    }
172}
173
174#[cfg(test)]
175mod tests {
176    use futures_util::{StreamExt as _, TryStreamExt as _};
177    use objectstore_types::metadata::{Compression, Metadata};
178
179    use super::maybe_decompress;
180    use crate::ClientStream;
181
182    fn compressed_zstd_stream(data: &[u8]) -> ClientStream {
183        let mut encoder = zstd::Encoder::new(vec![], 0).unwrap();
184        std::io::copy(&mut std::io::Cursor::new(data), &mut encoder).unwrap();
185        let compressed = encoder.finish().unwrap();
186        futures_util::stream::once(async move {
187            Ok::<_, std::io::Error>(bytes::Bytes::from(compressed))
188        })
189        .boxed()
190    }
191
192    fn raw_stream(data: &[u8]) -> ClientStream {
193        let bytes = bytes::Bytes::copy_from_slice(data);
194        futures_util::stream::once(async move { Ok::<_, std::io::Error>(bytes) }).boxed()
195    }
196
197    async fn collect(stream: ClientStream) -> Vec<u8> {
198        let chunks: bytes::BytesMut = stream.try_collect().await.unwrap();
199        chunks.to_vec()
200    }
201
202    fn zstd_metadata() -> Metadata {
203        Metadata {
204            compression: Some(Compression::Zstd),
205            ..Default::default()
206        }
207    }
208
209    fn no_compression_metadata() -> Metadata {
210        Metadata::default()
211    }
212
213    #[tokio::test]
214    async fn empty_accept_decompress_true_decompresses() {
215        let payload = b"hello world";
216        let stream = compressed_zstd_stream(payload);
217        let mut metadata = zstd_metadata();
218
219        let out = maybe_decompress(stream, &mut metadata, true, &[]);
220        assert_eq!(collect(out).await, payload);
221        assert_eq!(metadata.compression, None);
222    }
223
224    #[tokio::test]
225    async fn empty_accept_decompress_false_returns_compressed() {
226        let payload = b"hello world";
227        let compressed_bytes = collect(compressed_zstd_stream(payload)).await;
228        let stream = compressed_zstd_stream(payload);
229
230        let mut metadata = zstd_metadata();
231        let out = maybe_decompress(stream, &mut metadata, false, &[]);
232        assert_eq!(collect(out).await, compressed_bytes);
233        assert_eq!(metadata.compression, Some(Compression::Zstd));
234    }
235
236    #[tokio::test]
237    async fn zstd_accept_decompress_true_skips_decompression() {
238        let payload = b"hello world";
239        let compressed_bytes = collect(compressed_zstd_stream(payload)).await;
240        let stream = compressed_zstd_stream(payload);
241
242        let mut metadata = zstd_metadata();
243        let out = maybe_decompress(stream, &mut metadata, true, &[Compression::Zstd]);
244        assert_eq!(collect(out).await, compressed_bytes);
245        assert_eq!(metadata.compression, Some(Compression::Zstd));
246    }
247
248    #[tokio::test]
249    async fn zstd_accept_decompress_false_returns_compressed() {
250        let payload = b"hello world";
251        let compressed_bytes = collect(compressed_zstd_stream(payload)).await;
252        let stream = compressed_zstd_stream(payload);
253
254        let mut metadata = zstd_metadata();
255        let out = maybe_decompress(stream, &mut metadata, false, &[Compression::Zstd]);
256        assert_eq!(collect(out).await, compressed_bytes);
257        assert_eq!(metadata.compression, Some(Compression::Zstd));
258    }
259
260    #[tokio::test]
261    async fn no_compression_returns_raw_regardless_of_accept() {
262        let payload = b"hello world";
263        let stream = raw_stream(payload);
264
265        let mut metadata = no_compression_metadata();
266        let out = maybe_decompress(stream, &mut metadata, true, &[Compression::Zstd]);
267        assert_eq!(collect(out).await, payload);
268        assert_eq!(metadata.compression, None);
269    }
270
271    fn gzip_compress(data: &[u8]) -> Vec<u8> {
272        use std::io::Write as _;
273        let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
274        encoder.write_all(data).unwrap();
275        encoder.finish().unwrap()
276    }
277
278    fn gzip_metadata() -> Metadata {
279        Metadata {
280            compression: Some(Compression::Gzip),
281            ..Default::default()
282        }
283    }
284
285    #[tokio::test]
286    async fn gzip_decompression() {
287        let payload = b"hello world";
288        let stream = raw_stream(&gzip_compress(payload));
289        let mut metadata = gzip_metadata();
290
291        let out = maybe_decompress(stream, &mut metadata, true, &[]);
292        assert_eq!(collect(out).await, payload);
293        assert_eq!(metadata.compression, None);
294    }
295
296    #[tokio::test]
297    async fn gzip_accept_encoding() {
298        let compressed = gzip_compress(b"hello world");
299        let stream = raw_stream(&compressed);
300        let mut metadata = gzip_metadata();
301
302        let out = maybe_decompress(stream, &mut metadata, true, &[Compression::Gzip]);
303        assert_eq!(collect(out).await, compressed);
304        assert_eq!(metadata.compression, Some(Compression::Gzip));
305    }
306
307    #[tokio::test]
308    async fn gzip_concatenated_members() {
309        let mut compressed = gzip_compress(b"hello ");
310        compressed.extend(gzip_compress(b"world"));
311        let stream = raw_stream(&compressed);
312        let mut metadata = gzip_metadata();
313
314        let out = maybe_decompress(stream, &mut metadata, true, &[]);
315        assert_eq!(collect(out).await, b"hello world");
316        assert_eq!(metadata.compression, None);
317    }
318
319    #[tokio::test]
320    async fn zstd_concatenated_frames_decompress() {
321        let payload1 = b"hello ";
322        let payload2 = b"world";
323        let compressed1 = collect(compressed_zstd_stream(payload1)).await;
324        let compressed2 = collect(compressed_zstd_stream(payload2)).await;
325        let stream = futures_util::stream::iter([
326            Ok::<_, std::io::Error>(bytes::Bytes::from(compressed1)),
327            Ok::<_, std::io::Error>(bytes::Bytes::from(compressed2)),
328        ])
329        .boxed();
330
331        let mut metadata = zstd_metadata();
332        let out = maybe_decompress(stream, &mut metadata, true, &[]);
333        assert_eq!(collect(out).await, b"hello world");
334        assert_eq!(metadata.compression, None);
335    }
336}