1use std::{fmt, io};
2
3use async_compression::tokio::bufread::{GzipDecoder, ZstdDecoder};
4use bytes::BytesMut;
5use futures_util::{StreamExt, TryStreamExt};
6use objectstore_types::metadata::{Compression, Metadata};
7use reqwest::StatusCode;
8use tokio_util::io::{ReaderStream, StreamReader};
9
10use crate::instrument::{OperationSpan, record_field};
11use crate::response::ResponseExt as _;
12use crate::{ClientStream, ObjectKey, Session};
13
14pub struct GetResponse {
18 pub metadata: Metadata,
20 pub stream: ClientStream,
22}
23
24impl GetResponse {
25 pub async fn payload(self) -> crate::Result<bytes::Bytes> {
27 let bytes: BytesMut = self.stream.try_collect().await?;
28 Ok(bytes.freeze())
29 }
30
31 pub async fn text(self) -> crate::Result<String> {
33 let bytes = self.payload().await?;
34 Ok(String::from_utf8(bytes.to_vec())?)
35 }
36}
37
38impl fmt::Debug for GetResponse {
39 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
40 f.debug_struct("GetResponse")
41 .field("metadata", &self.metadata)
42 .field("stream", &format_args!("[Stream]"))
43 .finish()
44 }
45}
46
47impl Session {
48 pub fn get(&self, key: &str) -> GetBuilder {
50 GetBuilder {
51 session: self.clone(),
52 key: key.to_owned(),
53 decompress: true,
54 accept_encoding: vec![],
55 }
56 }
57}
58
59#[derive(Debug)]
61pub struct GetBuilder {
62 pub(crate) session: Session,
63 pub(crate) key: ObjectKey,
64 pub(crate) decompress: bool,
65 pub(crate) accept_encoding: Vec<Compression>,
66}
67
68impl GetBuilder {
69 pub fn decompress(mut self, decompress: bool) -> Self {
74 self.decompress = decompress;
75 self
76 }
77
78 pub fn accept_encoding(mut self, encodings: impl IntoIterator<Item = Compression>) -> Self {
85 self.accept_encoding = encodings.into_iter().collect();
86 self
87 }
88
89 pub async fn send(self) -> crate::Result<Option<GetResponse>> {
91 let span = OperationSpan::storage("objectstore.get", &self.session, Some(&self.key));
92 span.instrument(self.execute()).await
93 }
94
95 async fn execute(self) -> crate::Result<Option<GetResponse>> {
96 let response = self
97 .session
98 .request(reqwest::Method::GET, &self.key)?
99 .send()
100 .await?;
101 if response.status() == StatusCode::NOT_FOUND {
102 record_field("objectstore.found", false);
103 response.drain_body().await;
104 return Ok(None);
105 }
106 let response = response.error_for_status_and_drain().await?;
107 record_field("objectstore.found", true);
108
109 let mut metadata = match Metadata::from_headers(response.headers(), "") {
110 Ok(metadata) => metadata,
111 Err(error) => {
112 response.drain_body().await;
113 return Err(error.into());
114 }
115 };
116
117 let stored_compression = metadata.compression;
118 let stream = response.bytes_stream().map_err(io::Error::other).boxed();
119 let stream = maybe_decompress(
120 stream,
121 &mut metadata,
122 self.decompress,
123 &self.accept_encoding,
124 );
125 record_field(
126 "objectstore.compression",
127 stored_compression
128 .as_ref()
129 .map_or("none", Compression::as_str),
130 );
131 record_field(
132 "objectstore.decompressed",
133 metadata.compression != stored_compression,
134 );
135
136 Ok(Some(GetResponse { metadata, stream }))
137 }
138}
139
140pub(crate) fn maybe_decompress(
147 stream: ClientStream,
148 metadata: &mut Metadata,
149 decompress: bool,
150 accept_encoding: &[Compression],
151) -> ClientStream {
152 let encoding_accepted = metadata
153 .compression
154 .is_some_and(|c| accept_encoding.contains(&c));
155 match (metadata.compression, decompress && !encoding_accepted) {
156 (Some(Compression::Zstd), true) => {
157 metadata.compression = None;
158 let mut decoder = ZstdDecoder::new(StreamReader::new(stream));
159 decoder.multiple_members(true);
161 ReaderStream::new(decoder).boxed()
162 }
163 (Some(Compression::Gzip), true) => {
164 metadata.compression = None;
165 let mut decoder = GzipDecoder::new(StreamReader::new(stream));
166 decoder.multiple_members(true);
168 ReaderStream::new(decoder).boxed()
169 }
170 (Some(_), false) | (None, _) => stream,
171 }
172}
173
174#[cfg(test)]
175mod tests {
176 use futures_util::{StreamExt as _, TryStreamExt as _};
177 use objectstore_types::metadata::{Compression, Metadata};
178
179 use super::maybe_decompress;
180 use crate::ClientStream;
181
182 fn compressed_zstd_stream(data: &[u8]) -> ClientStream {
183 let mut encoder = zstd::Encoder::new(vec![], 0).unwrap();
184 std::io::copy(&mut std::io::Cursor::new(data), &mut encoder).unwrap();
185 let compressed = encoder.finish().unwrap();
186 futures_util::stream::once(async move {
187 Ok::<_, std::io::Error>(bytes::Bytes::from(compressed))
188 })
189 .boxed()
190 }
191
192 fn raw_stream(data: &[u8]) -> ClientStream {
193 let bytes = bytes::Bytes::copy_from_slice(data);
194 futures_util::stream::once(async move { Ok::<_, std::io::Error>(bytes) }).boxed()
195 }
196
197 async fn collect(stream: ClientStream) -> Vec<u8> {
198 let chunks: bytes::BytesMut = stream.try_collect().await.unwrap();
199 chunks.to_vec()
200 }
201
202 fn zstd_metadata() -> Metadata {
203 Metadata {
204 compression: Some(Compression::Zstd),
205 ..Default::default()
206 }
207 }
208
209 fn no_compression_metadata() -> Metadata {
210 Metadata::default()
211 }
212
213 #[tokio::test]
214 async fn empty_accept_decompress_true_decompresses() {
215 let payload = b"hello world";
216 let stream = compressed_zstd_stream(payload);
217 let mut metadata = zstd_metadata();
218
219 let out = maybe_decompress(stream, &mut metadata, true, &[]);
220 assert_eq!(collect(out).await, payload);
221 assert_eq!(metadata.compression, None);
222 }
223
224 #[tokio::test]
225 async fn empty_accept_decompress_false_returns_compressed() {
226 let payload = b"hello world";
227 let compressed_bytes = collect(compressed_zstd_stream(payload)).await;
228 let stream = compressed_zstd_stream(payload);
229
230 let mut metadata = zstd_metadata();
231 let out = maybe_decompress(stream, &mut metadata, false, &[]);
232 assert_eq!(collect(out).await, compressed_bytes);
233 assert_eq!(metadata.compression, Some(Compression::Zstd));
234 }
235
236 #[tokio::test]
237 async fn zstd_accept_decompress_true_skips_decompression() {
238 let payload = b"hello world";
239 let compressed_bytes = collect(compressed_zstd_stream(payload)).await;
240 let stream = compressed_zstd_stream(payload);
241
242 let mut metadata = zstd_metadata();
243 let out = maybe_decompress(stream, &mut metadata, true, &[Compression::Zstd]);
244 assert_eq!(collect(out).await, compressed_bytes);
245 assert_eq!(metadata.compression, Some(Compression::Zstd));
246 }
247
248 #[tokio::test]
249 async fn zstd_accept_decompress_false_returns_compressed() {
250 let payload = b"hello world";
251 let compressed_bytes = collect(compressed_zstd_stream(payload)).await;
252 let stream = compressed_zstd_stream(payload);
253
254 let mut metadata = zstd_metadata();
255 let out = maybe_decompress(stream, &mut metadata, false, &[Compression::Zstd]);
256 assert_eq!(collect(out).await, compressed_bytes);
257 assert_eq!(metadata.compression, Some(Compression::Zstd));
258 }
259
260 #[tokio::test]
261 async fn no_compression_returns_raw_regardless_of_accept() {
262 let payload = b"hello world";
263 let stream = raw_stream(payload);
264
265 let mut metadata = no_compression_metadata();
266 let out = maybe_decompress(stream, &mut metadata, true, &[Compression::Zstd]);
267 assert_eq!(collect(out).await, payload);
268 assert_eq!(metadata.compression, None);
269 }
270
271 fn gzip_compress(data: &[u8]) -> Vec<u8> {
272 use std::io::Write as _;
273 let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
274 encoder.write_all(data).unwrap();
275 encoder.finish().unwrap()
276 }
277
278 fn gzip_metadata() -> Metadata {
279 Metadata {
280 compression: Some(Compression::Gzip),
281 ..Default::default()
282 }
283 }
284
285 #[tokio::test]
286 async fn gzip_decompression() {
287 let payload = b"hello world";
288 let stream = raw_stream(&gzip_compress(payload));
289 let mut metadata = gzip_metadata();
290
291 let out = maybe_decompress(stream, &mut metadata, true, &[]);
292 assert_eq!(collect(out).await, payload);
293 assert_eq!(metadata.compression, None);
294 }
295
296 #[tokio::test]
297 async fn gzip_accept_encoding() {
298 let compressed = gzip_compress(b"hello world");
299 let stream = raw_stream(&compressed);
300 let mut metadata = gzip_metadata();
301
302 let out = maybe_decompress(stream, &mut metadata, true, &[Compression::Gzip]);
303 assert_eq!(collect(out).await, compressed);
304 assert_eq!(metadata.compression, Some(Compression::Gzip));
305 }
306
307 #[tokio::test]
308 async fn gzip_concatenated_members() {
309 let mut compressed = gzip_compress(b"hello ");
310 compressed.extend(gzip_compress(b"world"));
311 let stream = raw_stream(&compressed);
312 let mut metadata = gzip_metadata();
313
314 let out = maybe_decompress(stream, &mut metadata, true, &[]);
315 assert_eq!(collect(out).await, b"hello world");
316 assert_eq!(metadata.compression, None);
317 }
318
319 #[tokio::test]
320 async fn zstd_concatenated_frames_decompress() {
321 let payload1 = b"hello ";
322 let payload2 = b"world";
323 let compressed1 = collect(compressed_zstd_stream(payload1)).await;
324 let compressed2 = collect(compressed_zstd_stream(payload2)).await;
325 let stream = futures_util::stream::iter([
326 Ok::<_, std::io::Error>(bytes::Bytes::from(compressed1)),
327 Ok::<_, std::io::Error>(bytes::Bytes::from(compressed2)),
328 ])
329 .boxed();
330
331 let mut metadata = zstd_metadata();
332 let out = maybe_decompress(stream, &mut metadata, true, &[]);
333 assert_eq!(collect(out).await, b"hello world");
334 assert_eq!(metadata.compression, None);
335 }
336}