1#![cfg_attr(not(feature = "resumable-upload-api"), allow(dead_code))]
10
11use std::borrow::Cow;
12use std::collections::BTreeMap;
13use std::{fmt, io};
14
15use bytes::Bytes;
16use futures_util::{StreamExt, TryStreamExt, future, stream};
17use objectstore_types::metadata::Metadata;
18use objectstore_types::resumable::{
19 CompleteUploadResponse, CreateSessionResponse, HEADER_UPLOAD_LENGTH, HEADER_UPLOAD_OFFSET,
20 UploadOffset,
21};
22use reqwest::{Body, Method, Response, StatusCode};
23use serde::Serialize;
24use tokio::io::AsyncRead;
25use tokio_util::io::ReaderStream;
26
27pub use objectstore_types::resumable::{SessionToken, UploadProgress};
28
29use crate::instrument::{OperationSpan, record_field};
30use crate::response::ResponseExt as _;
31use crate::{ClientStream, Compression, Error, ExpirationPolicy, ObjectKey, Session};
32
33#[derive(Serialize)]
34#[serde(rename_all = "snake_case")]
35enum UploadType {
36 Resumable,
37}
38
39#[derive(Serialize)]
40struct UploadTypeQuery {
41 upload_type: UploadType,
42}
43
44#[derive(Serialize)]
45struct SessionQuery<'a> {
46 session: &'a SessionToken,
47}
48
49#[derive(Clone, Debug)]
54pub struct ResumableUpload {
55 session: Session,
56 key: ObjectKey,
57 token: SessionToken,
58 total_length: Option<u64>,
59 granularity: Option<u64>,
60}
61
62#[cfg(feature = "resumable-upload-api")]
63impl Session {
64 pub fn create_upload(&self, object_length: u64) -> CreateResumableUploadBuilder {
74 let metadata = Metadata {
75 expiration_policy: self.scope.usecase().expiration_policy(),
76 ..Default::default()
77 };
78
79 CreateResumableUploadBuilder::new(self.clone(), object_length, None, metadata)
80 }
81
82 pub fn resume_upload(&self, key: impl Into<ObjectKey>, token: SessionToken) -> ResumableUpload {
87 ResumableUpload::new(self.clone(), key.into(), token)
88 }
89}
90
91impl ResumableUpload {
92 fn new(session: Session, key: ObjectKey, token: SessionToken) -> Self {
93 Self {
94 session,
95 key,
96 token,
97 total_length: None,
98 granularity: None,
99 }
100 }
101
102 pub fn key(&self) -> &str {
104 &self.key
105 }
106
107 pub fn token(&self) -> &SessionToken {
109 &self.token
110 }
111
112 pub fn granularity(&self) -> Option<u64> {
117 self.granularity
118 }
119
120 pub fn progress(&self) -> UploadProgressBuilder {
122 UploadProgressBuilder {
123 upload: self.clone(),
124 }
125 }
126
127 pub fn put(&self, offset: u64, chunk: impl Into<Bytes>) -> PutChunkBuilder {
136 self.put_chunk(offset, UploadChunk::Buffer(chunk.into()))
137 }
138
139 pub fn put_stream(&self, offset: u64, length: u64, stream: ClientStream) -> PutChunkBuilder {
147 self.put_chunk(
148 offset,
149 UploadChunk::Stream {
150 source: stream,
151 length,
152 },
153 )
154 }
155
156 pub fn put_read<R>(&self, offset: u64, length: u64, reader: R) -> PutChunkBuilder
162 where
163 R: AsyncRead + Send + Sync + 'static,
164 {
165 self.put_stream(offset, length, ReaderStream::new(reader).boxed())
166 }
167
168 fn put_chunk(&self, offset: u64, chunk: UploadChunk) -> PutChunkBuilder {
169 PutChunkBuilder {
170 upload: self.clone(),
171 offset,
172 chunk,
173 }
174 }
175
176 fn validate_chunk_length(&self, offset: u64, chunk_length: u64) -> crate::Result<()> {
177 let (Some(granularity), Some(total_length)) = (self.granularity, self.total_length) else {
178 return Ok(());
181 };
182 if granularity == 0 || chunk_length == 0 || chunk_length >= granularity {
183 return Ok(());
184 }
185 if !offset
186 .checked_add(chunk_length)
187 .is_some_and(|end| end < total_length)
188 {
189 return Ok(());
190 }
191
192 Err(Error::ChunkTooSmall {
193 chunk_length,
194 upload_granularity: granularity,
195 })
196 }
197
198 pub fn cancel(&self) -> CancelUploadBuilder {
200 CancelUploadBuilder {
201 upload: self.clone(),
202 }
203 }
204
205 fn span(&self, op: &'static str) -> OperationSpan {
206 OperationSpan::storage(op, &self.session, Some(&self.key))
207 }
208
209 fn request(&self, method: Method) -> crate::Result<reqwest::RequestBuilder> {
210 Ok(self
211 .session
212 .request(method, &self.key)?
213 .query(&SessionQuery {
214 session: &self.token,
215 }))
216 }
217}
218
219#[derive(Debug)]
221pub struct CreateResumableUploadBuilder {
222 session: Session,
223 total_length: u64,
224 key: Option<ObjectKey>,
225 metadata: Metadata,
226}
227
228impl CreateResumableUploadBuilder {
229 pub(crate) fn new(
230 session: Session,
231 total_length: u64,
232 key: Option<ObjectKey>,
233 metadata: Metadata,
234 ) -> Self {
235 Self {
236 session,
237 total_length,
238 key,
239 metadata,
240 }
241 }
242
243 pub fn key(mut self, key: impl Into<ObjectKey>) -> Self {
245 self.key = Some(key.into()).filter(|key| !key.is_empty());
246 self
247 }
248
249 pub fn content_type(mut self, content_type: impl Into<Cow<'static, str>>) -> Self {
251 self.metadata.content_type = content_type.into();
252 self
253 }
254
255 pub fn expiration_policy(mut self, expiration_policy: ExpirationPolicy) -> Self {
257 self.metadata.expiration_policy = expiration_policy;
258 self
259 }
260
261 pub fn compression(mut self, compression: impl Into<Option<Compression>>) -> Self {
269 self.metadata.compression = compression.into();
270 self
271 }
272
273 pub fn origin(mut self, origin: impl Into<String>) -> Self {
275 self.metadata.origin = Some(origin.into());
276 self
277 }
278
279 pub fn filename(mut self, filename: impl Into<String>) -> Self {
281 self.metadata.filename = Some(filename.into());
282 self
283 }
284
285 pub fn set_metadata(mut self, metadata: impl Into<BTreeMap<String, String>>) -> Self {
290 self.metadata.custom = metadata.into();
291 self
292 }
293
294 pub fn append_metadata(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
300 self.metadata.custom.insert(key.into(), value.into());
301 self
302 }
303
304 pub async fn send(self) -> crate::Result<Option<ResumableUpload>> {
309 let span = OperationSpan::storage(
310 "objectstore.resumable.create",
311 &self.session,
312 self.key.as_deref(),
313 );
314 span.instrument(self.execute()).await
315 }
316
317 async fn execute(mut self) -> crate::Result<Option<ResumableUpload>> {
318 crate::metadata::normalize_keys(&mut self.metadata)?;
319 let method = if self.key.is_some() {
320 Method::PUT
321 } else {
322 Method::POST
323 };
324 let request = self
325 .session
326 .request(method, self.key.as_deref().unwrap_or_default())?
327 .query(&UploadTypeQuery {
328 upload_type: UploadType::Resumable,
329 })
330 .headers(self.metadata.to_headers("")?)
331 .header(HEADER_UPLOAD_LENGTH, self.total_length.to_string());
332 let response = request.send().await?;
333
334 match response.status() {
335 StatusCode::OK => {}
336 StatusCode::NOT_IMPLEMENTED => {
337 response.drain_body().await;
338 return Ok(None);
339 }
340 status => {
341 let response = response.error_for_status_and_drain().await?;
342 response.drain_body().await;
343 return Err(Error::MalformedResponse(format!(
344 "unexpected HTTP status {status} while creating a resumable upload"
345 )));
346 }
347 }
348
349 let response: CreateSessionResponse = response.json().await?;
350 record_field("objectstore.key", response.key.as_str());
351 let upload = ResumableUpload {
352 session: self.session,
353 key: response.key,
354 token: response.session,
355 total_length: Some(self.total_length),
356 granularity: Some(response.granularity),
357 };
358 Ok(Some(upload))
359 }
360}
361
362#[derive(Debug)]
364pub struct UploadProgressBuilder {
365 upload: ResumableUpload,
366}
367
368impl UploadProgressBuilder {
369 pub async fn send(self) -> crate::Result<UploadProgress> {
376 let span = self.upload.span("objectstore.resumable.progress");
377 span.instrument(self.execute()).await
378 }
379
380 async fn execute(self) -> crate::Result<UploadProgress> {
381 let response = self
382 .upload
383 .request(Method::PUT)?
384 .header(HEADER_UPLOAD_OFFSET, "*")
385 .send()
386 .await?;
387 parse_progress_response(response).await
388 }
389}
390
391pub struct PutChunkBuilder {
394 upload: ResumableUpload,
395 offset: u64,
396 chunk: UploadChunk,
397}
398
399enum UploadChunk {
401 Buffer(Bytes),
402 Stream { source: ClientStream, length: u64 },
403}
404
405impl UploadChunk {
406 fn len(&self) -> u64 {
407 match self {
408 Self::Buffer(bytes) => bytes.len() as u64,
409 Self::Stream { length, .. } => *length,
410 }
411 }
412
413 async fn into_body(self) -> io::Result<Body> {
416 let (stream, length) = match self {
417 Self::Buffer(bytes) => return Ok(Body::from(bytes)),
418 Self::Stream { source, length } => (source, length),
419 };
420 let stream = stream
421 .try_filter(|chunk| future::ready(!chunk.is_empty()))
422 .fuse();
423 let stream = stream::try_unfold((stream, length), |(mut stream, remaining)| async move {
424 let Some(chunk) = stream.try_next().await? else {
425 return if remaining == 0 {
426 Ok(None)
427 } else {
428 Err(io::Error::new(
429 io::ErrorKind::UnexpectedEof,
430 "upload stream is too short",
431 ))
432 };
433 };
434 let remaining = remaining.checked_sub(chunk.len() as u64).ok_or_else(|| {
435 io::Error::new(io::ErrorKind::InvalidData, "upload stream is too long")
436 })?;
437 if remaining == 0 && stream.try_next().await?.is_some() {
438 return Err(io::Error::new(
439 io::ErrorKind::InvalidData,
440 "upload stream is too long",
441 ));
442 }
443 Ok(Some((chunk, (stream, remaining))))
444 });
445
446 if length == 0 {
448 futures_util::pin_mut!(stream);
449 stream.try_next().await?;
450 return Ok(Body::default());
451 }
452 Ok(Body::wrap_stream(stream))
453 }
454}
455
456impl fmt::Debug for PutChunkBuilder {
457 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
458 f.debug_struct("PutChunkBuilder")
459 .field("upload", &self.upload)
460 .field("offset", &self.offset)
461 .field("length", &self.chunk.len())
462 .finish()
463 }
464}
465
466impl PutChunkBuilder {
467 pub async fn send(self) -> crate::Result<UploadProgress> {
488 let span = self.upload.span("objectstore.resumable.put");
489 span.instrument(self.execute()).await
490 }
491
492 async fn execute(self) -> crate::Result<UploadProgress> {
493 let length = self.chunk.len();
494 record_field("objectstore.offset", self.offset);
495 record_field("objectstore.size", length);
496 self.upload.validate_chunk_length(self.offset, length)?;
497 let body = self.chunk.into_body().await?;
498 let response = self
499 .upload
500 .request(Method::PUT)?
501 .header(HEADER_UPLOAD_OFFSET, self.offset.to_string())
502 .header(reqwest::header::CONTENT_LENGTH, length)
503 .body(body)
504 .send()
505 .await?;
506 parse_progress_response(response).await
507 }
508}
509
510#[derive(Debug)]
512pub struct CancelUploadBuilder {
513 upload: ResumableUpload,
514}
515
516impl CancelUploadBuilder {
517 pub async fn send(self) -> crate::Result<()> {
524 let span = self.upload.span("objectstore.resumable.cancel");
525 span.instrument(self.execute()).await
526 }
527
528 async fn execute(self) -> crate::Result<()> {
529 let response = self.upload.request(Method::DELETE)?.send().await?;
530 match response.status() {
531 StatusCode::NO_CONTENT => {
532 response.drain_body().await;
533 Ok(())
534 }
535 StatusCode::NOT_FOUND | StatusCode::GONE => {
536 response.drain_body().await;
537 Err(Error::ResumableUploadUnavailable)
538 }
539 status => {
540 let response = response.error_for_status_and_drain().await?;
541 response.drain_body().await;
542 Err(Error::MalformedResponse(format!(
543 "unexpected HTTP status {status} while canceling a resumable upload"
544 )))
545 }
546 }
547 }
548}
549
550async fn parse_progress_response(response: Response) -> crate::Result<UploadProgress> {
551 match response.status() {
552 status @ (StatusCode::NO_CONTENT | StatusCode::CONFLICT) => {
553 let offset = parse_offset(&response);
554 response.drain_body().await;
555 let offset = offset.ok_or_else(|| {
556 crate::Error::MalformedResponse(
557 "resumable upload response has no valid Upload-Offset header".into(),
558 )
559 })?;
560 if status == StatusCode::CONFLICT {
561 Err(Error::UploadOffsetMismatch { offset })
562 } else {
563 Ok(UploadProgress::Incomplete { offset })
564 }
565 }
566 StatusCode::CREATED => {
567 let _: CompleteUploadResponse = response.json().await?;
568 Ok(UploadProgress::Complete)
569 }
570 StatusCode::NOT_FOUND | StatusCode::GONE => {
571 response.drain_body().await;
572 Err(Error::ResumableUploadUnavailable)
573 }
574 status => {
575 let response = response.error_for_status_and_drain().await?;
576 response.drain_body().await;
577 Err(Error::MalformedResponse(format!(
578 "unexpected HTTP status {status} while continuing a resumable upload"
579 )))
580 }
581 }
582}
583
584fn parse_offset(response: &Response) -> Option<u64> {
585 let value = response
586 .headers()
587 .get(HEADER_UPLOAD_OFFSET)?
588 .to_str()
589 .ok()?;
590 match value.parse().ok()? {
591 UploadOffset::At(offset) => Some(offset),
592 UploadOffset::Unknown => None,
593 }
594}
595
596#[cfg(test)]
597mod tests {
598 use futures_util::FutureExt;
599 use http_body_util::BodyExt;
600
601 use super::*;
602
603 #[tokio::test]
604 async fn upload_chunk_checks_length() {
605 type Expected = Result<&'static [u8], ()>;
606 let cases: &[(u64, &[&[u8]], Expected)] = &[
607 (0, &[], Ok(b"")),
608 (0, &[b"", b""], Ok(b"")),
609 (0, &[b"x"], Err(())),
610 (5, &[], Err(())),
611 (5, &[b"hell"], Err(())),
612 (5, &[b"hello!"], Err(())),
613 (5, &[b"hello", b"!"], Err(())),
614 (5, &[b"hello"], Ok(b"hello")),
615 (5, &[b"", b"he", b"", b"llo", b""], Ok(b"hello")),
616 ];
617 for &(length, chunks, expected) in cases {
618 let stream = stream::iter(chunks.iter().map(|c| Ok(Bytes::from_static(c)))).boxed();
619 let result = async {
620 let body = UploadChunk::Stream {
621 source: stream,
622 length,
623 }
624 .into_body()
625 .await?;
626 Ok::<_, Error>(body.collect().await?.to_bytes())
627 }
628 .await;
629 assert_eq!(
630 result.as_deref().map_err(|_| ()),
631 expected,
632 "length={length}, chunks={chunks:?}: {result:?}"
633 );
634 }
635 }
636
637 #[tokio::test]
638 async fn upload_chunk_checks_eof() {
639 let (sender, receiver) = tokio::sync::mpsc::unbounded_channel();
640 let stream = stream::unfold(receiver, |mut receiver| async {
641 receiver.recv().await.map(|chunk| (Ok(chunk), receiver))
642 })
643 .boxed();
644 let mut body = UploadChunk::Stream {
645 source: stream,
646 length: 5,
647 }
648 .into_body()
649 .await
650 .unwrap();
651
652 sender.send(Bytes::from_static(b"he")).unwrap();
653 let first = body
654 .frame()
655 .now_or_never()
656 .expect("first chunk must not wait");
657 assert_eq!(first.unwrap().unwrap().into_data().unwrap(), b"he"[..]);
658
659 sender.send(Bytes::from_static(b"llo")).unwrap();
660 assert!(
661 body.frame().now_or_never().is_none(),
662 "final chunk must wait for EOF"
663 );
664 drop(sender);
665 assert_eq!(body.collect().await.unwrap().to_bytes(), b"llo"[..]);
666 }
667}