1use std::num::{NonZeroU32, NonZeroU64};
112use std::sync::Arc;
113use std::sync::atomic::Ordering;
114use std::time::Duration;
115
116use bytes::Bytes;
117use futures_util::StreamExt;
118use objectstore_metrics::TimerGuard;
119use objectstore_types::metadata::Metadata;
120use objectstore_types::range::ByteRange;
121use objectstore_types::resumable::UploadProgress;
122use objectstore_types::time::Timestamp;
123use sentry::{Hub, SentryFutureExt};
124use serde::{Deserialize, Serialize};
125
126use crate::backend::changelog::{Change, ChangeGuard, ChangeLog, ChangeManager, ChangePhase};
127use crate::backend::common::{
128 Backend, DeleteResponse, ExpiryTarget, ExpiryUpdate, GetResponse, HighVolumeBackend,
129 MetadataResponse, PutResponse, SetExpiryResponse, TieredGet, TieredMetadata, TieredUpdate,
130 TieredWrite, Tombstone,
131};
132use crate::backend::{HighVolumeStorageConfig, StorageConfig};
133use crate::concurrency::{Limiter, Permit, Priority, PriorityLimiter};
134use crate::error::{ErrorKind, Result, ResultExt as _};
135use crate::id::ObjectId;
136use crate::resumable::{BackendToken, Session};
137use crate::stream::{ClientStream, SizedPeek, counting_stream};
138
139const BACKEND_SIZE_THRESHOLD: usize = 1024 * 1024; const RESUMABLE_UPLOAD_TTL: Duration = Duration::from_hours(5 * 24);
144
145fn new_long_term_revision(id: &ObjectId) -> ObjectId {
151 ObjectId {
152 context: id.context.clone(),
153 key: format!("{}/{}", id.key, uuid::Uuid::now_v7()),
154 }
155}
156
157#[derive(Debug, Clone, Deserialize, Serialize)]
178pub struct TieredStorageConfig {
179 pub high_volume: HighVolumeStorageConfig,
184 pub long_term: Box<StorageConfig>,
186 pub concurrency: Option<TieredConcurrencyConfig>,
188}
189
190#[derive(Debug, Clone, Deserialize, Serialize)]
192pub struct TieredConcurrencyConfig {
193 pub high_volume: HighVolumeConcurrencyConfig,
195 pub long_term: LongTermConcurrencyConfig,
197}
198
199#[derive(Debug, Clone, Deserialize, Serialize)]
201pub struct HighVolumeConcurrencyConfig {
202 pub max_concurrency: NonZeroU32,
204 #[serde(default = "default_concurrency_timeout", with = "humantime_serde")]
206 pub timeout: Duration,
207 pub queue_size: Option<u32>,
210}
211
212#[derive(Debug, Clone, Deserialize, Serialize)]
214pub struct LongTermConcurrencyConfig {
215 pub max_concurrency: NonZeroU32,
217 #[serde(default, with = "humantime_serde")]
219 pub timeout: Duration,
220 pub queue_size: Option<u32>,
222}
223
224fn default_concurrency_timeout() -> Duration {
225 Duration::from_secs(1)
226}
227
228#[derive(Debug)]
290pub struct TieredStorage {
291 inner: Arc<ChangeManager>,
292 concurrency: Option<(PriorityLimiter, Limiter)>,
293}
294
295impl TieredStorage {
296 pub fn new(
298 high_volume: Box<dyn HighVolumeBackend>,
299 long_term: Box<dyn Backend>,
300 changelog: Box<dyn ChangeLog>,
301 ) -> Self {
302 let inner = ChangeManager::new(high_volume, long_term, changelog);
303 let hub = Hub::new_from_top(Hub::current());
304 tokio::spawn(inner.clone().recover().bind_hub(hub));
307 Self {
308 inner,
309 concurrency: None,
310 }
311 }
312
313 pub fn with_concurrency(mut self, config: TieredConcurrencyConfig) -> Self {
315 let high_volume = PriorityLimiter::new(
316 config.high_volume.max_concurrency,
317 config.high_volume.timeout,
318 None,
319 config.high_volume.queue_size,
320 None,
321 );
322 let long_term = Limiter::new(
323 config.long_term.max_concurrency,
324 config.long_term.timeout,
325 config.long_term.queue_size,
326 );
327 objectstore_metrics::gauge!(
328 "tiered.concurrency.limit" = config.high_volume.max_concurrency.get(),
329 tier = "high-volume"
330 );
331 objectstore_metrics::gauge!(
332 "tiered.concurrency.limit" = config.long_term.max_concurrency.get(),
333 tier = "long-term"
334 );
335 let emitter = high_volume.clone();
336 tokio::spawn(async move {
337 emitter
338 .run_emitter(|stats| async move {
339 objectstore_metrics::gauge!(
340 "tiered.concurrency.in_use" = stats.in_use,
341 tier = "high-volume"
342 );
343 objectstore_metrics::gauge!(
344 "tiered.concurrency.waiting" = stats.waiting_normal as u64,
345 tier = "high-volume",
346 priority = "normal"
347 );
348 objectstore_metrics::gauge!(
349 "tiered.concurrency.waiting" = stats.waiting_high as u64,
350 tier = "high-volume",
351 priority = "high"
352 );
353 })
354 .await;
355 });
356 let emitter = long_term.clone();
357 tokio::spawn(async move {
358 emitter
359 .run_emitter(|stats| async move {
360 objectstore_metrics::gauge!(
361 "tiered.concurrency.in_use" = stats.in_use,
362 tier = "long-term"
363 );
364 })
365 .await;
366 });
367 self.concurrency = Some((high_volume, long_term));
368 self
369 }
370
371 async fn acquire_high_volume(&self, priority: Priority) -> Result<Option<Permit>> {
372 let Some((limiter, _)) = &self.concurrency else {
373 return Ok(None);
374 };
375 let priority_tag = match priority {
376 Priority::Normal => "normal",
377 Priority::High => "high",
378 };
379 let timer = objectstore_metrics::timer!(
380 "tiered.concurrency.wait",
381 tier = "high-volume",
382 priority = priority_tag
383 );
384 let permit = limiter.acquire(priority).await.inspect_err(|_| {
385 objectstore_metrics::count!(
386 "tiered.concurrency.rejected",
387 tier = "high-volume",
388 priority = priority_tag
389 );
390 })?;
391 timer.record();
392 Ok(Some(permit))
393 }
394
395 async fn acquire_long_term(&self) -> Result<Option<Permit>> {
396 let Some((_, limiter)) = &self.concurrency else {
397 return Ok(None);
398 };
399 let timer = objectstore_metrics::timer!("tiered.concurrency.wait", tier = "long-term");
400 let permit = limiter.acquire().await.inspect_err(|_| {
401 objectstore_metrics::count!("tiered.concurrency.rejected", tier = "long-term");
402 })?;
403 timer.record();
404 Ok(Some(permit))
405 }
406
407 async fn record_change(&self, change: Change) -> Result<ChangeGuard> {
409 self.inner.clone().record(change).await
410 }
411
412 async fn check_upload_marker(&self, revision: &ObjectId) -> Result<()> {
414 if !self
415 .inner
416 .high_volume
417 .has_upload_marker(revision, Timestamp::now())
418 .await?
419 {
420 return Err(ErrorKind::UploadSessionGone.into());
421 }
422 Ok(())
423 }
424
425 async fn delete_upload_marker(&self, revision: &ObjectId) -> Result<()> {
427 if !self
428 .inner
429 .high_volume
430 .delete_upload_marker(revision, Timestamp::now())
431 .await?
432 {
433 return Err(ErrorKind::UploadSessionGone.into());
434 }
435 Ok(())
436 }
437
438 fn backend_type(&self, choice: &BackendChoice) -> &'static str {
440 match choice {
441 BackendChoice::HighVolume => self.inner.high_volume.name(),
442 BackendChoice::LongTerm => self.inner.long_term.name(),
443 }
444 }
445
446 fn record_chunk_size(&self, id: &ObjectId, content_length: u64) {
448 objectstore_metrics::record!(
449 "put_chunk.size" = content_length,
450 usecase = id.usecase().to_string(),
451 backend_choice = BackendChoice::LongTerm.as_str(),
452 backend_type = self.backend_type(&BackendChoice::LongTerm),
453 );
454 }
455
456 fn tag_backend(&self, timer: &mut TimerGuard, choice: &BackendChoice) {
458 timer.set_tag("backend_choice", choice.as_str());
459 timer.set_tag("backend_type", self.backend_type(choice));
460 }
461
462 #[tracing::instrument(level = "debug", fields(?id), skip_all)]
467 async fn put_high_volume(
468 &self,
469 id: &ObjectId,
470 metadata: &Metadata,
471 payload: Bytes,
472 access_time: Timestamp,
473 ) -> Result<()> {
474 let hv_guard = self.acquire_high_volume(Priority::Normal).await?;
475 let tombstone_opt = self
476 .inner
477 .high_volume
478 .put_non_tombstone(id, metadata, payload.clone(), access_time)
479 .await?;
480 drop(hv_guard);
481
482 let Some(Tombstone { target, .. }) = tombstone_opt else {
483 return Ok(());
485 };
486
487 let mut guard = self
489 .record_change(Change {
490 id: id.clone(),
491 new: None,
492 old: Some(target.clone()),
493 })
494 .await?;
495
496 let write = TieredWrite::Object(metadata.clone(), payload);
497 guard.advance(ChangePhase::Written);
498
499 let hv_guard = self.acquire_high_volume(Priority::Normal).await?;
500 let written = self
501 .inner
502 .high_volume
503 .compare_and_write(id, Some(&target), write, access_time)
504 .await?;
505
506 drop(hv_guard);
507 guard.advance(ChangePhase::compare_and_write(written));
509
510 Ok(())
511 }
512
513 #[tracing::instrument(level = "debug", fields(?id), skip_all)]
518 async fn put_long_term(
519 &self,
520 id: &ObjectId,
521 metadata: &Metadata,
522 stream: ClientStream,
523 access_time: Timestamp,
524 ) -> Result<()> {
525 let hv_guard = self.acquire_high_volume(Priority::Normal).await?;
526 let result = self
528 .inner
529 .high_volume
530 .get_tiered_metadata(id, access_time)
531 .await?;
532 drop(hv_guard);
533 let current = match result {
534 TieredMetadata::Tombstone(t) => Some(t.target),
535 _ => None,
536 };
537
538 let new = new_long_term_revision(id);
540 let mut guard = self
541 .record_change(Change {
542 id: id.clone(),
543 new: Some(new.clone()),
544 old: current.clone(),
545 })
546 .await?;
547
548 let lt_guard = self.acquire_long_term().await?;
549 self.inner
550 .long_term
551 .put_object(&new, metadata, stream, access_time)
552 .await?;
553 drop(lt_guard);
554 guard.advance(ChangePhase::Written);
555
556 let hv_guard = self.acquire_high_volume(Priority::High).await?;
557 let tombstone = Tombstone {
559 target: new.clone(),
560 time_expires: metadata.time_expires,
561 };
562 let written = self
563 .inner
564 .high_volume
565 .compare_and_write(
566 id,
567 current.as_ref(),
568 TieredWrite::Tombstone(tombstone),
569 access_time,
570 )
571 .await?;
572
573 drop(hv_guard);
575 guard.advance(ChangePhase::compare_and_write(written));
576
577 Ok(())
578 }
579}
580
581type LongTermBackendToken = BackendToken;
582
583#[derive(Debug, Serialize, Deserialize)]
584struct TieredResumableToken {
585 inner: LongTermBackendToken,
586 revision: String,
587 time_expires: Option<Timestamp>,
588}
589
590impl TieredResumableToken {
591 fn decode(token: &BackendToken) -> Result<Self> {
592 serde_json::from_str(token).map_err(|_| ErrorKind::UnknownUploadSession.into())
593 }
594
595 fn into_inner_session(self, session: &Session) -> Session {
597 Session {
598 object_id: ObjectId {
599 context: session.object_id.context.clone(),
600 key: self.revision,
601 },
602 backend_token: self.inner,
603 ..session.clone()
604 }
605 }
606}
607
608#[async_trait::async_trait]
609impl Backend for TieredStorage {
610 fn name(&self) -> &'static str {
611 "tiered"
612 }
613
614 fn upload_granularity(&self) -> u64 {
615 self.inner.long_term.upload_granularity()
616 }
617
618 #[tracing::instrument(level = "debug", fields(?id, upload_length), skip_all)]
619 async fn create_upload_session(
620 &self,
621 id: &ObjectId,
622 metadata: &Metadata,
623 upload_length: NonZeroU64,
624 ) -> Result<Option<BackendToken>> {
625 if upload_length.get() <= BACKEND_SIZE_THRESHOLD as u64 {
626 return Ok(None);
627 }
628
629 let revision = new_long_term_revision(id);
630 let lt_guard = self.acquire_long_term().await?;
631 let result = self
632 .inner
633 .long_term
634 .create_upload_session(&revision, metadata, upload_length)
635 .await?;
636 drop(lt_guard);
637 let Some(inner) = result else {
638 return Ok(None);
639 };
640
641 let hv_guard = self.acquire_high_volume(Priority::High).await?;
642 self.inner
643 .high_volume
644 .create_upload_marker(&revision, Timestamp::now() + RESUMABLE_UPLOAD_TTL)
645 .await?;
646 drop(hv_guard);
647
648 if metadata.origin.is_none() {
649 objectstore_metrics::count!("put.origin_missing", usecase = id.usecase().to_string());
650 }
651 let token = TieredResumableToken {
652 revision: revision.key,
653 inner,
654 time_expires: metadata.time_expires,
655 };
656 Ok(Some(serde_json::to_string(&token).context(
657 ErrorKind::Internal,
658 "encoding tiered resumable session",
659 )?))
660 }
661
662 #[tracing::instrument(level = "debug", fields(?session, offset, content_length), skip_all)]
663 async fn put_chunk(
664 &self,
665 session: &Session,
666 offset: u64,
667 content_length: u64,
668 stream: ClientStream,
669 ) -> Result<UploadProgress> {
670 let timer = objectstore_metrics::timer!(
671 "put_chunk.latency",
672 usecase = session.object_id.usecase().to_string(),
673 backend_choice = BackendChoice::LongTerm.as_str(),
674 backend_type = self.backend_type(&BackendChoice::LongTerm),
675 );
676 let tiered = TieredResumableToken::decode(&session.backend_token)?;
677 let time_expires = tiered.time_expires;
678 let inner_session = tiered.into_inner_session(session);
679 let id = &session.object_id;
680 let revision = &inner_session.object_id;
681 let end = offset
682 .checked_add(content_length)
683 .filter(|end| *end <= session.upload_length.get())
684 .ok_or(ErrorKind::ChunkExceedsUploadLength {
685 offset,
686 content_length,
687 upload_length: session.upload_length.get(),
688 })?;
689
690 let granularity = self.upload_granularity();
691 if content_length > 0 && content_length < granularity && end != session.upload_length.get()
692 {
693 return Err(ErrorKind::ChunkTooSmall {
694 chunk_length: content_length,
695 upload_granularity: granularity,
696 }
697 .into());
698 }
699
700 if end != session.upload_length.get() {
702 let lt_guard = self.acquire_long_term().await?;
703 let progress = self
704 .inner
705 .long_term
706 .put_chunk(&inner_session, offset, content_length, stream)
707 .await?;
708 drop(lt_guard);
709 if progress == UploadProgress::Complete {
710 return Err(ErrorKind::UploadSessionGone.into());
711 }
712 timer.record();
713 self.record_chunk_size(id, content_length);
714 return Ok(progress);
715 }
716
717 let (progress, current) = tokio::join!(
719 async {
720 let lt_guard = self.acquire_long_term().await?;
721 let result = self.inner.long_term.upload_offset(&inner_session).await;
722 drop(lt_guard);
723 result
724 },
725 async {
726 let hv_guard = self.acquire_high_volume(Priority::High).await?;
727 let result = self
728 .inner
729 .high_volume
730 .get_tiered_metadata(id, Timestamp::now())
731 .await;
732 drop(hv_guard);
733 result
734 },
735 );
736 match progress? {
737 UploadProgress::Complete => return Err(ErrorKind::UploadSessionGone.into()),
738 UploadProgress::Incomplete { offset: actual } if actual < offset => {
739 return Err(ErrorKind::UploadOffsetMismatch { offset: actual }.into());
740 }
741 UploadProgress::Incomplete { .. } => {}
742 }
743 let current = match current? {
744 TieredMetadata::Tombstone(t) if t.target == *revision => {
746 timer.record();
747 return Ok(UploadProgress::Complete);
748 }
749 TieredMetadata::Tombstone(t) => Some(t.target),
750 _ => None,
751 };
752
753 let lt_guard = self.acquire_long_term().await?;
755 let hv_guard = self.acquire_high_volume(Priority::High).await?;
756 self.delete_upload_marker(revision).await?;
757 drop(hv_guard);
758
759 let mut guard = self
761 .record_change(Change {
762 id: id.clone(),
763 new: Some(revision.clone()),
764 old: current.clone(),
765 })
766 .await?;
767
768 let progress = self
769 .inner
770 .long_term
771 .put_chunk(&inner_session, offset, content_length, stream)
772 .await?;
773 drop(lt_guard);
774 if progress != UploadProgress::Complete {
775 return Err(ErrorKind::UploadSessionGone.into());
776 }
777 guard.advance(ChangePhase::Written);
778
779 let hv_guard = self.acquire_high_volume(Priority::High).await?;
780 let written = self
781 .inner
782 .high_volume
783 .compare_and_write(
784 id,
785 current.as_ref(),
786 TieredWrite::Tombstone(Tombstone {
787 target: revision.clone(),
788 time_expires,
789 }),
790 Timestamp::now(),
791 )
792 .await?;
793 drop(hv_guard);
794 guard.advance(ChangePhase::compare_and_write(written));
795
796 timer.record();
797 self.record_chunk_size(id, content_length);
798 objectstore_metrics::record!(
799 "put.size" = session.upload_length.get(),
800 usecase = id.usecase().to_string(),
801 backend_choice = BackendChoice::LongTerm.as_str(),
802 backend_type = self.backend_type(&BackendChoice::LongTerm),
803 upload_type = "resumable",
804 );
805 Ok(UploadProgress::Complete)
806 }
807
808 #[tracing::instrument(level = "debug", fields(?session), skip_all)]
809 async fn upload_offset(&self, session: &Session) -> Result<UploadProgress> {
810 let tiered = TieredResumableToken::decode(&session.backend_token)?;
811 let inner_session = tiered.into_inner_session(session);
812 let hv_guard = self.acquire_high_volume(Priority::Normal).await?;
813 self.check_upload_marker(&inner_session.object_id).await?;
814 drop(hv_guard);
815 let lt_guard = self.acquire_long_term().await?;
816 let result = self.inner.long_term.upload_offset(&inner_session).await?;
817 drop(lt_guard);
818 match result {
819 UploadProgress::Incomplete { offset } => Ok(UploadProgress::Incomplete { offset }),
820 UploadProgress::Complete => Err(ErrorKind::UploadSessionGone.into()),
821 }
822 }
823
824 #[tracing::instrument(level = "debug", fields(?session), skip_all)]
825 async fn cancel_upload(&self, session: &Session) -> Result<()> {
826 let tiered = TieredResumableToken::decode(&session.backend_token)?;
827 let inner_session = tiered.into_inner_session(session);
828 let hv_guard = self.acquire_high_volume(Priority::Normal).await?;
829 self.delete_upload_marker(&inner_session.object_id).await?;
830 drop(hv_guard);
831 let result = async {
832 let lt_guard = self.acquire_long_term().await?;
833 let result = self.inner.long_term.cancel_upload(&inner_session).await;
834 drop(lt_guard);
835 result
836 }
837 .await;
838 if let Err(error) = result {
839 objectstore_log::warn!(!!&error, "Failed to cancel upload after deleting marker");
840 }
841 Ok(())
842 }
843
844 #[tracing::instrument(level = "debug", fields(?id), skip_all)]
845 async fn put_object(
846 &self,
847 id: &ObjectId,
848 metadata: &Metadata,
849 stream: ClientStream,
850 access_time: Timestamp,
851 ) -> Result<PutResponse> {
852 let mut timer =
853 objectstore_metrics::timer!("put.latency", usecase = id.usecase().to_string(),);
854 if metadata.origin.is_none() {
855 objectstore_metrics::count!("put.origin_missing", usecase = id.usecase().to_string());
856 }
857
858 let peeked = SizedPeek::new(stream, BACKEND_SIZE_THRESHOLD).await?;
859 objectstore_metrics::record!(
860 "put.first_chunk.latency" = timer.elapsed(),
861 usecase = id.usecase().to_string(),
862 complete = if peeked.is_exhausted() { "yes" } else { "no" },
863 );
864
865 let (backend_choice, stored_size) = if peeked.is_exhausted() {
866 self.tag_backend(&mut timer, &BackendChoice::HighVolume);
867 let payload = peeked.into_bytes().await?;
868 let payload_len = payload.len() as u64;
869 self.put_high_volume(id, metadata, payload, access_time)
870 .await?;
871 (BackendChoice::HighVolume, payload_len)
872 } else {
873 self.tag_backend(&mut timer, &BackendChoice::LongTerm);
874 let (stored_size, stream) = counting_stream(peeked.into_stream());
875 self.put_long_term(id, metadata, stream.boxed(), access_time)
876 .await?;
877 (BackendChoice::LongTerm, stored_size.load(Ordering::Acquire))
878 };
879
880 let backend_ty = self.backend_type(&backend_choice);
881 timer.record();
882 objectstore_metrics::record!(
883 "put.size" = stored_size,
884 usecase = id.usecase().to_string(),
885 backend_choice = backend_choice.as_str(),
886 backend_type = backend_ty,
887 upload_type = "direct",
888 );
889
890 Ok(())
891 }
892
893 #[tracing::instrument(level = "debug", skip(self))]
894 async fn get_object(
895 &self,
896 id: &ObjectId,
897 access_time: Timestamp,
898 range: Option<ByteRange>,
899 ) -> Result<GetResponse> {
900 let mut timer = objectstore_metrics::timer!(
901 "get.latency.pre-response",
902 usecase = id.usecase().to_string(),
903 );
904 self.tag_backend(&mut timer, &BackendChoice::HighVolume);
905
906 let hv_guard = self.acquire_high_volume(Priority::Normal).await?;
907 let hv_result = self
908 .inner
909 .high_volume
910 .get_tiered_object(id, access_time, range)
911 .await?;
912 drop(hv_guard);
913 let (result, backend_choice) = match hv_result {
914 TieredGet::NotFound => (None, BackendChoice::HighVolume),
915 TieredGet::Object(metadata, content_range, stream) => (
916 Some((metadata, content_range, stream)),
917 BackendChoice::HighVolume,
918 ),
919 TieredGet::Tombstone(tombstone) => {
920 self.tag_backend(&mut timer, &BackendChoice::LongTerm);
921 let lt_guard = self.acquire_long_term().await?;
922 let result = self
923 .inner
924 .long_term
925 .get_object(&tombstone.target, access_time, range)
926 .await?
927 .map(|(meta, range, stream)| {
928 let stream = match lt_guard {
929 Some(permit) => permit.guard_stream(stream).boxed(),
930 None => stream,
931 };
932 (align_expiry(meta, &tombstone), range, stream)
933 });
934 (result, BackendChoice::LongTerm)
935 }
936 };
937
938 let backend_type = self.backend_type(&backend_choice);
939 timer.record();
940
941 if let Some((ref metadata, ref content_range, _)) = result {
942 let size = content_range.map(|cr| cr.len() as usize).or(metadata.size);
943 if let Some(size) = size {
944 objectstore_metrics::record!(
945 "get.size" = size,
946 usecase = id.usecase().to_string(),
947 backend_choice = backend_choice.as_str(),
948 backend_type = backend_type,
949 );
950 }
951 }
952
953 Ok(result)
954 }
955
956 #[tracing::instrument(level = "debug", skip(self))]
957 async fn get_metadata(
958 &self,
959 id: &ObjectId,
960 access_time: Timestamp,
961 ) -> Result<MetadataResponse> {
962 let mut timer =
963 objectstore_metrics::timer!("head.latency", usecase = id.usecase().to_string());
964 self.tag_backend(&mut timer, &BackendChoice::HighVolume);
965
966 let hv_guard = self.acquire_high_volume(Priority::Normal).await?;
967 let hv_result = self
968 .inner
969 .high_volume
970 .get_tiered_metadata(id, access_time)
971 .await?;
972 drop(hv_guard);
973 let result = match hv_result {
974 TieredMetadata::NotFound => None,
975 TieredMetadata::Object(metadata) => Some(metadata),
976 TieredMetadata::Tombstone(tombstone) => {
977 self.tag_backend(&mut timer, &BackendChoice::LongTerm);
978 let lt_guard = self.acquire_long_term().await?;
979 let result = self
980 .inner
981 .long_term
982 .get_metadata(&tombstone.target, access_time)
983 .await?;
984 drop(lt_guard);
985 result.map(|metadata| align_expiry(metadata, &tombstone))
986 }
987 };
988
989 timer.set_tag("result", if result.is_some() { "hit" } else { "miss" });
990 timer.record();
991
992 Ok(result)
993 }
994
995 async fn set_expiry(
996 &self,
997 id: &ObjectId,
998 target: ExpiryUpdate,
999 access_time: Timestamp,
1000 ) -> Result<SetExpiryResponse> {
1001 let hv_guard = self.acquire_high_volume(Priority::Normal).await?;
1002 let result = self
1003 .inner
1004 .high_volume
1005 .get_tiered_metadata(id, access_time)
1006 .await?;
1007 drop(hv_guard);
1008 match result {
1009 TieredMetadata::NotFound => Ok(SetExpiryResponse::NotFound),
1010 TieredMetadata::Object(_) => {
1011 let hv_guard = self.acquire_high_volume(Priority::Normal).await?;
1012 let result = self
1013 .inner
1014 .high_volume
1015 .compare_and_update(id, None, TieredUpdate::SetExpiry(target), access_time)
1016 .await;
1017 drop(hv_guard);
1018 result
1019 }
1020 TieredMetadata::Tombstone(tombstone) => {
1021 let lt_guard = self.acquire_long_term().await?;
1022 let result = self
1025 .inner
1026 .long_term
1027 .set_expiry(&tombstone.target, target, access_time)
1028 .await?;
1029 drop(lt_guard);
1030 let deadline = match result {
1031 SetExpiryResponse::Satisfied(deadline) => deadline,
1032 outcome @ (SetExpiryResponse::NotFound | SetExpiryResponse::Rejected) => {
1033 return Ok(outcome);
1034 }
1035 };
1036
1037 let hv_guard = self.acquire_high_volume(Priority::High).await?;
1038 let update = TieredUpdate::SetExpiry(ExpiryTarget::At(deadline).into());
1040
1041 let result = self
1047 .inner
1048 .high_volume
1049 .compare_and_update(id, Some(&tombstone.target), update, access_time)
1050 .await;
1051 drop(hv_guard);
1052 result
1053 }
1054 }
1055 }
1056
1057 #[tracing::instrument(level = "debug", skip(self))]
1058 async fn delete_object(&self, id: &ObjectId, access_time: Timestamp) -> Result<DeleteResponse> {
1059 let mut timer =
1060 objectstore_metrics::timer!("delete.latency", usecase = id.usecase().to_string());
1061 self.tag_backend(&mut timer, &BackendChoice::HighVolume);
1062
1063 let hv_guard = self.acquire_high_volume(Priority::Normal).await?;
1064 let result = self
1065 .inner
1066 .high_volume
1067 .delete_non_tombstone(id, access_time)
1068 .await?;
1069 drop(hv_guard);
1070 if let Some(tombstone) = result {
1071 self.tag_backend(&mut timer, &BackendChoice::LongTerm);
1072
1073 let mut guard = self
1074 .record_change(Change {
1075 id: id.clone(),
1076 new: None,
1077 old: Some(tombstone.target.clone()),
1078 })
1079 .await?;
1080 guard.advance(ChangePhase::Written);
1081
1082 let hv_guard = self.acquire_high_volume(Priority::Normal).await?;
1084 let deleted = self
1085 .inner
1086 .high_volume
1087 .compare_and_write(
1088 id,
1089 Some(&tombstone.target),
1090 TieredWrite::Delete,
1091 access_time,
1092 )
1093 .await?;
1094
1095 drop(hv_guard);
1096 guard.advance(ChangePhase::compare_and_write(deleted));
1098 }
1099
1100 timer.record();
1101
1102 Ok(())
1103 }
1104
1105 async fn join(&self) {
1106 self.inner.tracker.close();
1107 tokio::join!(
1108 self.inner.high_volume.join(),
1109 self.inner.long_term.join(),
1110 self.inner.tracker.wait()
1111 );
1112 }
1113}
1114
1115#[derive(Debug)]
1116enum BackendChoice {
1117 HighVolume,
1118 LongTerm,
1119}
1120
1121impl BackendChoice {
1122 fn as_str(&self) -> &'static str {
1123 match self {
1124 BackendChoice::HighVolume => "high-volume",
1125 BackendChoice::LongTerm => "long-term",
1126 }
1127 }
1128}
1129
1130impl std::fmt::Display for BackendChoice {
1131 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1132 f.write_str(self.as_str())
1133 }
1134}
1135
1136fn effective_expiry(
1140 redirect_expiry: Option<Timestamp>,
1141 blob_expiry: Option<Timestamp>,
1142) -> Option<Timestamp> {
1143 match (redirect_expiry, blob_expiry) {
1144 (Some(redirect), Some(blob)) => Some(redirect.min(blob)),
1145 (Some(expiry), None) | (None, Some(expiry)) => Some(expiry),
1146 (None, None) => None,
1147 }
1148}
1149
1150fn align_expiry(mut metadata: Metadata, tombstone: &Tombstone) -> Metadata {
1155 metadata.time_expires = effective_expiry(tombstone.time_expires, metadata.time_expires);
1156 metadata
1157}
1158
1159#[cfg(test)]
1160mod tests {
1161 use std::num::NonZeroU64;
1162 use std::sync::Mutex as StdMutex;
1163
1164 use objectstore_types::metadata::{ExpirationPolicy, Metadata};
1165 use objectstore_types::scope::{Scope, Scopes};
1166
1167 use super::*;
1168 use crate::backend::bigtable::{BigTableBackend, BigTableConfig};
1169 use crate::backend::changelog::{InMemoryChangeLog, NoopChangeLog};
1170 use crate::backend::gcs::{GcsBackend, GcsConfig};
1171 use crate::backend::in_memory::InMemoryBackend;
1172 use crate::backend::testing::{Hooks, TestBackend};
1173 use crate::change_stream::ChangeStreamFactory;
1174 use crate::error::Error;
1175 use crate::id::ObjectContext;
1176 use crate::stream::{self, ClientStream};
1177
1178 #[test]
1179 fn tiered_concurrency_config() {
1180 let leaf = serde_json::json!({"type":"bigtable", "project_id":"test", "instance_name":"test", "table_name":"test"});
1181 let mut config = serde_json::json!({"high_volume":leaf, "long_term":leaf});
1182 assert!(
1183 serde_json::from_value::<TieredStorageConfig>(config.clone())
1184 .unwrap()
1185 .concurrency
1186 .is_none()
1187 );
1188 for invalid in [
1189 serde_json::json!({}),
1190 serde_json::json!({"high_volume":{"max_concurrency":1}}),
1191 serde_json::json!({"high_volume":{"max_concurrency":0},"long_term":{"max_concurrency":1}}),
1192 ] {
1193 config["concurrency"] = invalid;
1194 assert!(serde_json::from_value::<TieredStorageConfig>(config.clone()).is_err());
1195 }
1196 config["concurrency"] = serde_json::json!({
1197 "high_volume": {"max_concurrency": 100, "timeout": "250ms", "queue_size": 30},
1198 "long_term": {"max_concurrency": 50, "timeout": "3s", "queue_size": 30}
1199 });
1200 let limits = serde_json::from_value::<TieredStorageConfig>(config)
1201 .unwrap()
1202 .concurrency
1203 .unwrap();
1204 assert_eq!(limits.high_volume.timeout, Duration::from_millis(250));
1205 assert_eq!(limits.long_term.timeout, Duration::from_secs(3));
1206 assert_eq!(limits.high_volume.queue_size, Some(30));
1207 assert_eq!(limits.long_term.queue_size, Some(30));
1208 }
1209
1210 fn make_context() -> ObjectContext {
1211 ObjectContext {
1212 usecase: "testing".parse().unwrap(),
1213 scopes: Scopes::from_iter([Scope::create("testing", "value").unwrap()]),
1214 }
1215 }
1216
1217 fn make_id(key: &str) -> ObjectId {
1218 ObjectId::new(make_context(), key.into())
1219 }
1220
1221 fn make_tiered_storage() -> (
1222 TieredStorage,
1223 InMemoryBackend,
1224 InMemoryBackend,
1225 InMemoryChangeLog,
1226 ) {
1227 let hv = InMemoryBackend::new("in-memory-hv");
1228 let lt = InMemoryBackend::new("in-memory-lt");
1229 let changelog = InMemoryChangeLog::default();
1230 let storage = TieredStorage::new(
1231 Box::new(hv.clone()),
1232 Box::new(lt.clone()),
1233 Box::new(changelog.clone()),
1234 );
1235 (storage, hv, lt, changelog)
1236 }
1237
1238 async fn resumable_token(
1239 storage: &TieredStorage,
1240 id: &ObjectId,
1241 metadata: &Metadata,
1242 length: u64,
1243 ) -> Session {
1244 let backend_token = storage
1245 .create_upload_session(id, metadata, NonZeroU64::new(length).unwrap())
1246 .await
1247 .unwrap()
1248 .unwrap();
1249 Session {
1250 object_id: id.clone(),
1251 upload_length: NonZeroU64::new(length).unwrap(),
1252 backend_token,
1253 }
1254 }
1255
1256 fn upload_revision(session: &Session) -> ObjectId {
1257 TieredResumableToken::decode(&session.backend_token)
1258 .unwrap()
1259 .into_inner_session(session)
1260 .object_id
1261 }
1262
1263 #[tokio::test]
1264 async fn resumable_inmemory() -> anyhow::Result<()> {
1265 let (storage, _, _, _) = make_tiered_storage();
1266 let id = make_id("tiered-resumable-inmemory");
1267 let payload = vec![b'a'; BACKEND_SIZE_THRESHOLD + 1];
1268 let token =
1269 resumable_token(&storage, &id, &Metadata::default(), payload.len() as u64).await;
1270
1271 let revision = upload_revision(&token);
1272 assert!(
1273 storage
1274 .inner
1275 .high_volume
1276 .has_upload_marker(&revision, Timestamp::now())
1277 .await?
1278 );
1279 assert_eq!(
1280 storage.upload_offset(&token).await?,
1281 UploadProgress::Incomplete { offset: 0 }
1282 );
1283 let split = 256 * 1024;
1284 assert_eq!(
1285 storage
1286 .put_chunk(
1287 &token,
1288 0,
1289 split as u64,
1290 stream::single(payload[..split].to_vec())
1291 )
1292 .await?,
1293 UploadProgress::Incomplete {
1294 offset: split as u64
1295 }
1296 );
1297 assert!(
1298 storage
1299 .inner
1300 .high_volume
1301 .has_upload_marker(&revision, Timestamp::now())
1302 .await?
1303 );
1304 assert_eq!(
1305 storage.upload_offset(&token).await?,
1306 UploadProgress::Incomplete {
1307 offset: split as u64
1308 }
1309 );
1310 assert!(storage.get_metadata(&id, Timestamp::now()).await?.is_none());
1311
1312 assert_eq!(
1314 storage
1315 .put_chunk(
1316 &token,
1317 split as u64,
1318 (payload.len() - split) as u64,
1319 stream::single(payload[split..].to_vec())
1320 )
1321 .await?,
1322 UploadProgress::Complete
1323 );
1324 assert!(
1325 !storage
1326 .inner
1327 .high_volume
1328 .has_upload_marker(&revision, Timestamp::now())
1329 .await?
1330 );
1331 assert_eq!(
1332 storage.upload_offset(&token).await.unwrap_err().kind(),
1333 ErrorKind::UploadSessionGone
1334 );
1335 let (_, _, body) = storage
1336 .get_object(&id, Timestamp::now(), None)
1337 .await?
1338 .unwrap();
1339 assert_eq!(stream::read_to_vec(body).await?, payload);
1340 Ok(())
1341 }
1342
1343 #[tokio::test]
1344 async fn resumable_bigtable_and_gcs() -> anyhow::Result<()> {
1345 let streams = ChangeStreamFactory::default();
1346 let lt = GcsBackend::new(
1347 GcsConfig {
1348 endpoint: Some("http://localhost:8087".into()),
1349 bucket: "test-bucket".into(),
1350 tcp_user_timeout: Duration::from_secs(30),
1351 cogs: None,
1352 },
1353 &streams,
1354 )
1355 .await?;
1356 let hv = BigTableBackend::new(
1357 BigTableConfig {
1358 endpoint: Some("localhost:8086".into()),
1359 project_id: "testing".into(),
1360 instance_name: "objectstore".into(),
1361 table_name: "objectstore".into(),
1362 connections: None,
1363 rpc_timeout: Duration::from_secs(2),
1364 cogs: None,
1365 },
1366 &streams,
1367 )
1368 .await?;
1369 let storage = TieredStorage::new(Box::new(hv), Box::new(lt), Box::new(NoopChangeLog));
1370 let id = make_id(&format!("tiered-resumable-{}", uuid::Uuid::now_v7()));
1371 let payload = vec![b'a'; BACKEND_SIZE_THRESHOLD + 1];
1372 let token =
1373 resumable_token(&storage, &id, &Metadata::default(), payload.len() as u64).await;
1374
1375 let error = storage
1376 .put_chunk(&token, 0, 1, stream::single("a"))
1377 .await
1378 .unwrap_err();
1379 assert_eq!(
1380 error.kind(),
1381 ErrorKind::ChunkTooSmall {
1382 chunk_length: 1,
1383 upload_granularity: 256 * 1024,
1384 }
1385 );
1386
1387 let revision = upload_revision(&token);
1388 assert!(
1389 storage
1390 .inner
1391 .high_volume
1392 .has_upload_marker(&revision, Timestamp::now())
1393 .await?
1394 );
1395 assert_eq!(
1396 storage.upload_offset(&token).await?,
1397 UploadProgress::Incomplete { offset: 0 }
1398 );
1399 let split = 256 * 1024;
1400 assert_eq!(
1401 storage
1402 .put_chunk(
1403 &token,
1404 0,
1405 split as u64,
1406 stream::single(payload[..split].to_vec())
1407 )
1408 .await?,
1409 UploadProgress::Incomplete {
1410 offset: split as u64
1411 }
1412 );
1413 assert!(
1414 storage
1415 .inner
1416 .high_volume
1417 .has_upload_marker(&revision, Timestamp::now())
1418 .await?
1419 );
1420 assert_eq!(
1421 storage.upload_offset(&token).await?,
1422 UploadProgress::Incomplete {
1423 offset: split as u64
1424 }
1425 );
1426 assert!(storage.get_metadata(&id, Timestamp::now()).await?.is_none());
1427
1428 assert_eq!(
1430 storage
1431 .put_chunk(
1432 &token,
1433 split as u64,
1434 (payload.len() - split) as u64,
1435 stream::single(payload[split..].to_vec())
1436 )
1437 .await?,
1438 UploadProgress::Complete
1439 );
1440 assert!(
1441 !storage
1442 .inner
1443 .high_volume
1444 .has_upload_marker(&revision, Timestamp::now())
1445 .await?
1446 );
1447 assert_eq!(
1448 storage.upload_offset(&token).await.unwrap_err().kind(),
1449 ErrorKind::UploadSessionGone
1450 );
1451 let (_, _, body) = storage
1452 .get_object(&id, Timestamp::now(), None)
1453 .await?
1454 .unwrap();
1455 assert_eq!(stream::read_to_vec(body).await?, payload);
1456 Ok(())
1457 }
1458
1459 #[tokio::test]
1460 async fn resumable_invalid_chunks() {
1461 let (storage, _, _, _) = make_tiered_storage();
1462 let id = make_id("resumable-invalid");
1463 let length = BACKEND_SIZE_THRESHOLD as u64 + 1;
1464 let token = resumable_token(&storage, &id, &Metadata::default(), length).await;
1465
1466 assert_eq!(
1468 storage
1469 .put_chunk(&token, u64::MAX, 1, stream::single("x"))
1470 .await
1471 .unwrap_err()
1472 .kind(),
1473 ErrorKind::ChunkExceedsUploadLength {
1474 offset: u64::MAX,
1475 content_length: 1,
1476 upload_length: length
1477 }
1478 );
1479 assert_eq!(
1480 storage
1481 .put_chunk(&token, length - 1, 1, stream::single("x"))
1482 .await
1483 .unwrap_err()
1484 .kind(),
1485 ErrorKind::UploadOffsetMismatch { offset: 0 }
1486 );
1487 }
1488
1489 #[tokio::test]
1490 async fn resumable_cancel() {
1491 let (storage, hv, _, _) = make_tiered_storage();
1492 let id = make_id("resumable-invalid");
1493 let token = resumable_token(
1494 &storage,
1495 &id,
1496 &Metadata::default(),
1497 BACKEND_SIZE_THRESHOLD as u64 + 1,
1498 )
1499 .await;
1500
1501 let revision = upload_revision(&token);
1502 assert!(
1503 hv.has_upload_marker(&revision, Timestamp::now())
1504 .await
1505 .unwrap()
1506 );
1507 storage.cancel_upload(&token).await.unwrap();
1508 assert!(
1509 !hv.has_upload_marker(&revision, Timestamp::now())
1510 .await
1511 .unwrap()
1512 );
1513 assert_eq!(
1514 storage.cancel_upload(&token).await.unwrap_err().kind(),
1515 ErrorKind::UploadSessionGone
1516 );
1517 assert_eq!(
1518 storage.upload_offset(&token).await.unwrap_err().kind(),
1519 ErrorKind::UploadSessionGone
1520 );
1521 assert!(!hv.contains(&id));
1522 }
1523
1524 #[derive(Clone, Debug)]
1525 struct ExpiryHook {
1526 label: &'static str,
1527 events: Arc<StdMutex<Vec<&'static str>>>,
1528 reject: bool,
1529 }
1530
1531 type ExpiryTestStorage = (
1532 TieredStorage,
1533 TestBackend<ExpiryHook>,
1534 TestBackend<ExpiryHook>,
1535 Arc<StdMutex<Vec<&'static str>>>,
1536 );
1537
1538 #[async_trait::async_trait]
1539 impl Hooks for ExpiryHook {
1540 async fn set_expiry(
1541 &self,
1542 inner: &InMemoryBackend,
1543 id: &ObjectId,
1544 target: ExpiryUpdate,
1545 access_time: Timestamp,
1546 ) -> Result<SetExpiryResponse> {
1547 self.events.lock().unwrap().push(self.label);
1548 if self.reject {
1549 Ok(SetExpiryResponse::Rejected)
1550 } else {
1551 inner.set_expiry(id, target, access_time).await
1552 }
1553 }
1554
1555 async fn compare_and_update(
1556 &self,
1557 inner: &InMemoryBackend,
1558 id: &ObjectId,
1559 current: Option<&ObjectId>,
1560 update: TieredUpdate,
1561 access_time: Timestamp,
1562 ) -> Result<SetExpiryResponse> {
1563 self.events.lock().unwrap().push(self.label);
1564 if self.reject {
1565 Ok(SetExpiryResponse::Rejected)
1566 } else {
1567 inner
1568 .compare_and_update(id, current, update, access_time)
1569 .await
1570 }
1571 }
1572 }
1573
1574 fn tiered_with_expiry_hooks(hv_reject: bool, lt_reject: bool) -> ExpiryTestStorage {
1575 let events = Arc::new(StdMutex::new(Vec::new()));
1576 let hv = TestBackend::new(ExpiryHook {
1577 label: "hv",
1578 events: Arc::clone(&events),
1579 reject: hv_reject,
1580 });
1581 let lt = TestBackend::new(ExpiryHook {
1582 label: "lt",
1583 events: Arc::clone(&events),
1584 reject: lt_reject,
1585 });
1586 let storage = TieredStorage::new(
1587 Box::new(hv.clone()),
1588 Box::new(lt.clone()),
1589 Box::new(NoopChangeLog),
1590 );
1591 (storage, hv, lt, events)
1592 }
1593
1594 async fn seed_redirect(
1595 hv: &InMemoryBackend,
1596 lt: &InMemoryBackend,
1597 id: &ObjectId,
1598 target: &ObjectId,
1599 expiry: Timestamp,
1600 ) {
1601 let metadata = Metadata {
1602 expiration_policy: ExpirationPolicy::TimeToLive(Duration::from_mins(10)),
1603 time_created: expiry.checked_sub(Duration::from_mins(10)),
1604 time_expires: Some(expiry),
1605 ..Default::default()
1606 };
1607 lt.put_object(
1608 target,
1609 &metadata,
1610 stream::single("payload"),
1611 Timestamp::now(),
1612 )
1613 .await
1614 .unwrap();
1615 hv.compare_and_write(
1616 id,
1617 None,
1618 TieredWrite::Tombstone(Tombstone {
1619 target: target.clone(),
1620 time_expires: metadata.time_expires,
1621 }),
1622 Timestamp::now(),
1623 )
1624 .await
1625 .unwrap();
1626 }
1627
1628 #[tokio::test]
1629 async fn set_expiry() {
1630 let (storage, hv, lt, events) = tiered_with_expiry_hooks(false, false);
1631 let id = make_id("tiered-expiry-order");
1632 let target = new_long_term_revision(&id);
1633 let old_expiry = Timestamp::now() + Duration::from_mins(10);
1634 seed_redirect(&hv.inner, <.inner, &id, &target, old_expiry).await;
1635
1636 let created = old_expiry - Duration::from_mins(10);
1637 let requested = created + Duration::from_hours(1);
1638 let blob_expiry = requested + Duration::from_hours(1);
1639 lt.inner
1640 .put_object(
1641 &target,
1642 &Metadata {
1643 expiration_policy: ExpirationPolicy::TimeToIdle(Duration::from_hours(1)),
1644 time_created: Some(created),
1645 time_expires: Some(blob_expiry),
1646 ..Default::default()
1647 },
1648 stream::single("payload"),
1649 Timestamp::now(),
1650 )
1651 .await
1652 .unwrap();
1653 let update = ExpiryUpdate {
1654 target: ExpiryTarget::FromCreation(Duration::from_hours(1)),
1655 max: Some(Duration::ZERO),
1656 };
1657 assert_eq!(
1658 storage
1659 .set_expiry(&id, update, created)
1660 .await
1661 .unwrap_err()
1662 .kind(),
1663 ErrorKind::InvalidMetadata,
1664 );
1665 assert_eq!(events.lock().unwrap().as_slice(), &["lt"]);
1666 events.lock().unwrap().clear();
1667 assert_eq!(
1668 hv.inner.get(&id).expect_tombstone().time_expires,
1669 Some(old_expiry)
1670 );
1671 assert_eq!(
1672 storage
1673 .set_expiry(
1674 &id,
1675 ExpiryUpdate {
1676 max: Some(Duration::from_hours(1)),
1677 ..update
1678 },
1679 Timestamp::now(),
1680 )
1681 .await
1682 .unwrap(),
1683 SetExpiryResponse::Satisfied(requested)
1684 );
1685 assert_eq!(events.lock().unwrap().as_slice(), &["lt", "hv"]);
1686 assert_eq!(
1687 lt.inner.get(&target).expect_object().0.time_expires,
1688 Some(blob_expiry)
1689 );
1690 assert_eq!(
1691 hv.inner.get(&id).expect_tombstone().time_expires,
1692 Some(requested)
1693 );
1694 }
1695
1696 #[derive(Clone, Debug)]
1697 struct ReplaceInlineOnLookup {
1698 replacement: Metadata,
1699 replaced: Arc<StdMutex<bool>>,
1700 }
1701
1702 #[async_trait::async_trait]
1703 impl Hooks for ReplaceInlineOnLookup {
1704 async fn get_tiered_metadata(
1705 &self,
1706 inner: &InMemoryBackend,
1707 id: &ObjectId,
1708 access_time: Timestamp,
1709 ) -> Result<TieredMetadata> {
1710 let observed = inner.get_tiered_metadata(id, access_time).await?;
1711 let should_replace = {
1712 let mut replaced = self.replaced.lock().unwrap();
1713 !std::mem::replace(&mut *replaced, true)
1714 };
1715 if should_replace {
1716 inner
1717 .put_object(
1718 id,
1719 &self.replacement,
1720 stream::single("replacement"),
1721 access_time,
1722 )
1723 .await?;
1724 }
1725 Ok(observed)
1726 }
1727 }
1728
1729 #[tokio::test]
1730 async fn inline_creation_target_uses_update_snapshot() {
1731 let access_time = Timestamp::from_unix_secs(1_700_000_000).unwrap();
1732 let old_created = access_time - Duration::from_hours(2);
1733 let new_created = access_time - Duration::from_mins(30);
1734 let old_expiry = access_time + Duration::from_mins(10);
1735 let resolved = new_created + Duration::from_hours(2);
1736 let replacement = Metadata {
1737 expiration_policy: ExpirationPolicy::TimeToLive(Duration::from_hours(1)),
1738 time_created: Some(new_created),
1739 time_expires: Some(old_expiry),
1740 ..Default::default()
1741 };
1742 let hv = TestBackend::new(ReplaceInlineOnLookup {
1743 replacement,
1744 replaced: Arc::new(StdMutex::new(false)),
1745 });
1746 let storage = TieredStorage::new(
1747 Box::new(hv.clone()),
1748 Box::new(InMemoryBackend::new("lt")),
1749 Box::new(NoopChangeLog),
1750 );
1751 let id = make_id("inline-update-snapshot");
1752 hv.inner
1753 .put_object(
1754 &id,
1755 &Metadata {
1756 expiration_policy: ExpirationPolicy::TimeToLive(Duration::from_hours(1)),
1757 time_created: Some(old_created),
1758 time_expires: Some(old_expiry),
1759 ..Default::default()
1760 },
1761 stream::single("original"),
1762 access_time,
1763 )
1764 .await
1765 .unwrap();
1766
1767 assert_eq!(
1768 storage
1769 .set_expiry(
1770 &id,
1771 ExpiryTarget::FromCreation(Duration::from_hours(2)).into(),
1772 access_time,
1773 )
1774 .await
1775 .unwrap(),
1776 SetExpiryResponse::Satisfied(resolved)
1777 );
1778 let (metadata, payload) = hv.inner.get(&id).expect_object();
1779 assert_eq!(
1780 metadata.expiration_policy,
1781 ExpirationPolicy::TimeToLive(Duration::from_hours(2))
1782 );
1783 assert_eq!(metadata.time_created, Some(new_created));
1784 assert_eq!(metadata.time_expires, Some(resolved));
1785 assert_eq!(payload, Bytes::from_static(b"replacement"));
1786 }
1787
1788 #[tokio::test]
1789 async fn expiry_hv_conflict() {
1790 let (storage, hv, lt, events) = tiered_with_expiry_hooks(true, false);
1791 let id = make_id("tiered-hv-failure");
1792 let target = new_long_term_revision(&id);
1793 let old_expiry = Timestamp::now() + Duration::from_mins(10);
1794 seed_redirect(&hv.inner, <.inner, &id, &target, old_expiry).await;
1795
1796 let requested = old_expiry + Duration::from_mins(50);
1797 assert_eq!(
1798 storage
1799 .set_expiry(
1800 &id,
1801 ExpiryTarget::FromCreation(Duration::from_hours(1)).into(),
1802 Timestamp::now(),
1803 )
1804 .await
1805 .unwrap(),
1806 SetExpiryResponse::Rejected
1807 );
1808 assert_eq!(events.lock().unwrap().as_slice(), &["lt", "hv"]);
1809 assert_eq!(
1810 hv.inner.get(&id).expect_tombstone().time_expires,
1811 Some(old_expiry)
1812 );
1813 assert_eq!(
1814 lt.inner.get(&target).expect_object().0.time_expires,
1815 Some(requested)
1816 );
1817 assert_eq!(
1818 storage
1819 .get_metadata(&id, Timestamp::now())
1820 .await
1821 .unwrap()
1822 .unwrap()
1823 .time_expires,
1824 Some(old_expiry)
1825 );
1826 assert_eq!(
1827 lt.inner.get(&target).expect_object().0.expiration_policy,
1828 ExpirationPolicy::TimeToLive(Duration::from_hours(1))
1829 );
1830 }
1831
1832 #[tokio::test]
1833 async fn expiry_lt_conflict() {
1834 let (storage, hv, lt, events) = tiered_with_expiry_hooks(false, true);
1835 let id = make_id("tiered-lt-conflict");
1836 let target = new_long_term_revision(&id);
1837 seed_redirect(
1838 &hv.inner,
1839 <.inner,
1840 &id,
1841 &target,
1842 Timestamp::now() + Duration::from_mins(10),
1843 )
1844 .await;
1845
1846 assert_eq!(
1847 storage
1848 .set_expiry(
1849 &id,
1850 ExpiryTarget::At(Timestamp::now() + Duration::from_hours(1)).into(),
1851 Timestamp::now()
1852 )
1853 .await
1854 .unwrap(),
1855 SetExpiryResponse::Rejected
1856 );
1857 assert_eq!(events.lock().unwrap().as_slice(), &["lt"]);
1858 }
1859
1860 #[tokio::test]
1861 async fn expiry_not_found() {
1862 for missing_blob in [false, true] {
1863 let (storage, hv, lt, events) = tiered_with_expiry_hooks(false, false);
1864 let id = make_id("tiered-expiry-missing");
1865 let target = new_long_term_revision(&id);
1866 let access_time = Timestamp::now();
1867 let deadline = access_time + Duration::from_hours(1);
1868 if missing_blob {
1869 seed_redirect(&hv.inner, <.inner, &id, &target, deadline).await;
1870 lt.inner.delete_object(&target, access_time).await.unwrap();
1871 }
1872
1873 assert_eq!(
1874 storage
1875 .set_expiry(&id, ExpiryTarget::At(deadline).into(), access_time)
1876 .await
1877 .unwrap(),
1878 SetExpiryResponse::NotFound
1879 );
1880 let expected: &[&str] = if missing_blob { &["lt"] } else { &[] };
1881 assert_eq!(events.lock().unwrap().as_slice(), expected);
1882 }
1883 }
1884
1885 #[test]
1888 fn revision_id_preserves_context() {
1889 let id = make_id("my-key");
1890 let revised = new_long_term_revision(&id);
1891 assert_eq!(revised.context, id.context);
1892 assert!(
1893 revised.key.starts_with("my-key/"),
1894 "revised key should have /<uuid> suffix, got: {}",
1895 revised.key
1896 );
1897 }
1898
1899 #[test]
1900 fn revision_id_roundtrips_storage_path() {
1901 let id = make_id("original");
1902 let revised = new_long_term_revision(&id);
1903 let path = revised.as_storage_path().to_string();
1904 let parsed = ObjectId::from_storage_path(&path)
1905 .unwrap_or_else(|| panic!("failed to parse '{path}'"));
1906 assert_eq!(parsed, revised);
1907 }
1908
1909 #[test]
1910 fn revision_id_is_unique() {
1911 let id = make_id("base-key");
1912 let a = new_long_term_revision(&id);
1913 let b = new_long_term_revision(&id);
1914 assert_ne!(a.key, b.key, "two calls should produce different keys");
1915 }
1916
1917 #[tokio::test]
1920 async fn get_nonexistent_returns_none() {
1921 let (storage, _hv, _lt, _) = make_tiered_storage();
1922 let id = make_id("does-not-exist");
1923
1924 assert!(
1925 storage
1926 .get_object(&id, Timestamp::now(), None)
1927 .await
1928 .unwrap()
1929 .is_none()
1930 );
1931 assert!(
1932 storage
1933 .get_metadata(&id, Timestamp::now())
1934 .await
1935 .unwrap()
1936 .is_none()
1937 );
1938 }
1939
1940 #[tokio::test]
1941 async fn delete_nonexistent_succeeds() {
1942 let (storage, _hv, _lt, _) = make_tiered_storage();
1943 let id = make_id("does-not-exist");
1944
1945 storage.delete_object(&id, Timestamp::now()).await.unwrap();
1946 }
1947
1948 #[tokio::test]
1951 async fn put_small_object_stores_inline() {
1952 let (storage, hv, lt, _) = make_tiered_storage();
1953 let id = make_id("small");
1954 let payload = b"small payload".to_vec();
1955
1956 storage
1957 .put_object(
1958 &id,
1959 &Metadata::default(),
1960 stream::single(payload.clone()),
1961 Timestamp::now(),
1962 )
1963 .await
1964 .unwrap();
1965
1966 assert!(hv.contains(&id), "expected in high-volume");
1967 assert!(!lt.contains(&id), "leaked to long-term");
1968
1969 let (_, _, s) = storage
1970 .get_object(&id, Timestamp::now(), None)
1971 .await
1972 .unwrap()
1973 .unwrap();
1974 let body = stream::read_to_vec(s).await.unwrap();
1975 assert_eq!(body, payload);
1976
1977 assert!(
1978 storage
1979 .get_metadata(&id, Timestamp::now())
1980 .await
1981 .unwrap()
1982 .is_some(),
1983 "get_metadata should return metadata for inline objects"
1984 );
1985 }
1986
1987 #[tokio::test]
1988 async fn put_large_object_creates_tombstone() {
1989 let (storage, hv, lt, _) = make_tiered_storage();
1990 let id = make_id("large");
1991 let payload = vec![0xCDu8; 2 * 1024 * 1024]; let metadata_in = Metadata {
1993 content_type: "image/png".into(),
1994 expiration_policy: ExpirationPolicy::TimeToLive(Duration::from_hours(1)),
1995 time_expires: Some(Timestamp::now() + Duration::from_hours(1)),
1996 origin: Some("10.0.0.1".into()),
1997 ..Metadata::default()
1998 };
1999
2000 storage
2001 .put_object(
2002 &id,
2003 &metadata_in,
2004 stream::single(payload.clone()),
2005 Timestamp::now(),
2006 )
2007 .await
2008 .unwrap();
2009
2010 let tombstone = hv.get(&id).expect_tombstone();
2012 assert_eq!(tombstone.time_expires, metadata_in.time_expires);
2013 let lt_id = tombstone.target;
2014 assert!(
2015 lt_id.key().starts_with(id.key()),
2016 "tombstone target key should be a revision of the HV key, got: {}",
2017 lt_id.key()
2018 );
2019
2020 let (lt_meta, _) = lt.get(<_id).expect_object();
2022 assert_eq!(lt_meta.content_type, "image/png");
2023 assert_eq!(lt_meta.expiration_policy, metadata_in.expiration_policy);
2024 assert_eq!(lt_meta.time_expires, tombstone.time_expires);
2025
2026 let (_, _, s) = storage
2028 .get_object(&id, Timestamp::now(), None)
2029 .await
2030 .unwrap()
2031 .unwrap();
2032 let body = stream::read_to_vec(s).await.unwrap();
2033 assert_eq!(body, payload);
2034
2035 let metadata = storage
2037 .get_metadata(&id, Timestamp::now())
2038 .await
2039 .unwrap()
2040 .unwrap();
2041 assert_eq!(metadata.content_type, "image/png");
2042 }
2043
2044 #[tokio::test]
2047 async fn reinsert_small_over_large_swaps_to_inline() {
2048 let (storage, hv, lt, _) = make_tiered_storage();
2049 let id = make_id("reinsert-key");
2050
2051 let large_payload = vec![0xABu8; 2 * 1024 * 1024];
2053 storage
2054 .put_object(
2055 &id,
2056 &Metadata::default(),
2057 stream::single(large_payload),
2058 Timestamp::now(),
2059 )
2060 .await
2061 .unwrap();
2062
2063 let lt_id = hv.get(&id).expect_tombstone().target;
2064
2065 let small_payload = vec![0xCDu8; 100]; storage
2069 .put_object(
2070 &id,
2071 &Metadata::default(),
2072 stream::single(small_payload),
2073 Timestamp::now(),
2074 )
2075 .await
2076 .unwrap();
2077
2078 hv.get(&id).expect_object();
2080
2081 storage.join().await;
2083
2084 lt.get(<_id).expect_not_found();
2086 }
2087
2088 #[tokio::test]
2089 async fn overwrite_large_with_large_replaces_revision() {
2090 let (storage, hv, lt, _) = make_tiered_storage();
2091 let id = make_id("overwrite-large");
2092
2093 let payload1 = vec![0xAAu8; 2 * 1024 * 1024];
2094 storage
2095 .put_object(
2096 &id,
2097 &Metadata::default(),
2098 stream::single(payload1),
2099 Timestamp::now(),
2100 )
2101 .await
2102 .unwrap();
2103 let lt_id_1 = hv.get(&id).expect_tombstone().target;
2104
2105 let payload2 = vec![0xBBu8; 2 * 1024 * 1024];
2106 storage
2107 .put_object(
2108 &id,
2109 &Metadata::default(),
2110 stream::single(payload2.clone()),
2111 Timestamp::now(),
2112 )
2113 .await
2114 .unwrap();
2115 let lt_id_2 = hv.get(&id).expect_tombstone().target;
2116
2117 assert_ne!(
2118 lt_id_1, lt_id_2,
2119 "second write should create a new revision"
2120 );
2121
2122 storage.join().await;
2124
2125 lt.get(<_id_1).expect_not_found();
2126 lt.get(<_id_2).expect_object();
2127
2128 let (_, _, s) = storage
2129 .get_object(&id, Timestamp::now(), None)
2130 .await
2131 .unwrap()
2132 .unwrap();
2133 let body = stream::read_to_vec(s).await.unwrap();
2134 assert_eq!(body, payload2);
2135 }
2136
2137 #[tokio::test]
2140 async fn delete_small_object() {
2141 let (storage, hv, _lt, _) = make_tiered_storage();
2142 let id = make_id("delete-small");
2143
2144 storage
2145 .put_object(
2146 &id,
2147 &Metadata::default(),
2148 stream::single("tiny"),
2149 Timestamp::now(),
2150 )
2151 .await
2152 .unwrap();
2153
2154 storage.delete_object(&id, Timestamp::now()).await.unwrap();
2155
2156 hv.get(&id).expect_not_found();
2157 assert!(
2158 storage
2159 .get_object(&id, Timestamp::now(), None)
2160 .await
2161 .unwrap()
2162 .is_none()
2163 );
2164 }
2165
2166 #[tokio::test]
2167 async fn delete_large_object_cleans_up_both_backends() {
2168 let (storage, hv, lt, _) = make_tiered_storage();
2169 let id = make_id("delete-both");
2170 let payload = vec![0u8; 2 * 1024 * 1024]; storage
2173 .put_object(
2174 &id,
2175 &Metadata::default(),
2176 stream::single(payload),
2177 Timestamp::now(),
2178 )
2179 .await
2180 .unwrap();
2181
2182 let lt_id = hv.get(&id).expect_tombstone().target;
2184
2185 storage.delete_object(&id, Timestamp::now()).await.unwrap();
2186
2187 storage.join().await;
2189
2190 assert!(!hv.contains(&id), "tombstone not cleaned up");
2191 assert!(!lt.contains(<_id), "long-term object not cleaned up");
2192 }
2193
2194 #[derive(Debug)]
2195 struct FailDelete;
2196
2197 #[async_trait::async_trait]
2198 impl Hooks for FailDelete {
2199 async fn delete_object(
2200 &self,
2201 _inner: &InMemoryBackend,
2202 _id: &ObjectId,
2203 _access_time: Timestamp,
2204 ) -> Result<DeleteResponse> {
2205 Err(Error::with_source(
2206 ErrorKind::BackendFailure,
2207 std::io::Error::new(
2208 std::io::ErrorKind::ConnectionRefused,
2209 "simulated long-term delete failure",
2210 ),
2211 ))
2212 }
2213 }
2214
2215 #[tokio::test]
2219 async fn delete_succeeds_when_gcs_cleanup_fails() {
2220 let hv = InMemoryBackend::new("hv");
2221 let lt = TestBackend::new(FailDelete);
2222 let log = NoopChangeLog;
2223 let storage = TieredStorage::new(Box::new(hv.clone()), Box::new(lt), Box::new(log));
2224
2225 let id = make_id("fail-delete");
2226 let payload = vec![0xABu8; 2 * 1024 * 1024]; storage
2228 .put_object(
2229 &id,
2230 &Metadata::default(),
2231 stream::single(payload),
2232 Timestamp::now(),
2233 )
2234 .await
2235 .unwrap();
2236
2237 let result = storage.delete_object(&id, Timestamp::now()).await;
2239 assert!(
2240 result.is_ok(),
2241 "delete should succeed despite GCS cleanup failure"
2242 );
2243
2244 hv.get(&id).expect_not_found();
2246
2247 assert!(
2249 storage
2250 .get_object(&id, Timestamp::now(), None)
2251 .await
2252 .unwrap()
2253 .is_none(),
2254 "object should be unreachable after tombstone is deleted"
2255 );
2256 }
2257
2258 #[derive(Debug, Copy, Clone)]
2261 struct CasConflict;
2262
2263 #[async_trait::async_trait]
2264 impl Hooks for CasConflict {
2265 async fn compare_and_write(
2266 &self,
2267 _inner: &InMemoryBackend,
2268 _id: &ObjectId,
2269 _current: Option<&ObjectId>,
2270 _write: TieredWrite,
2271 _access_time: Timestamp,
2272 ) -> Result<bool> {
2273 Ok(false) }
2275 }
2276
2277 #[tokio::test]
2281 async fn put_large_cas_conflict_cleans_up_new_blob() {
2282 let hv = TestBackend::new(CasConflict);
2283 let lt = InMemoryBackend::new("lt");
2284 let log = NoopChangeLog;
2285 let storage = TieredStorage::new(Box::new(hv), Box::new(lt.clone()), Box::new(log));
2286
2287 let id = make_id("cas-conflict-large");
2288 let payload = vec![0xABu8; 2 * 1024 * 1024]; storage
2291 .put_object(
2292 &id,
2293 &Metadata::default(),
2294 stream::single(payload),
2295 Timestamp::now(),
2296 )
2297 .await
2298 .unwrap();
2299
2300 storage.join().await;
2302
2303 assert!(
2304 lt.is_empty(),
2305 "LT blob should be cleaned up after CAS conflict"
2306 );
2307 }
2308
2309 #[tokio::test]
2310 async fn upload_cas_conflict_cleans_up_new_blob() {
2311 let hv = TestBackend::new(CasConflict);
2312 let lt = InMemoryBackend::new("lt");
2313 let storage =
2314 TieredStorage::new(Box::new(hv), Box::new(lt.clone()), Box::new(NoopChangeLog));
2315 let id = make_id("upload-cas-conflict");
2316 let payload = vec![0xAB; BACKEND_SIZE_THRESHOLD + 1];
2317 let session =
2318 resumable_token(&storage, &id, &Metadata::default(), payload.len() as u64).await;
2319
2320 assert_eq!(
2321 storage
2322 .put_chunk(&session, 0, payload.len() as u64, stream::single(payload))
2323 .await
2324 .unwrap(),
2325 UploadProgress::Complete
2326 );
2327 storage.join().await;
2328 assert!(
2329 lt.is_empty(),
2330 "LT blob should be cleaned up after CAS conflict"
2331 );
2332 }
2333
2334 #[tokio::test]
2338 async fn put_small_over_tombstone_cas_conflict_succeeds() {
2339 let inner = InMemoryBackend::new("hv");
2340 let id = make_id("cas-conflict-small");
2341
2342 let tombstone = Tombstone {
2345 target: make_id("lt-object"),
2346 time_expires: None,
2347 };
2348 inner
2349 .compare_and_write(
2350 &id,
2351 None,
2352 TieredWrite::Tombstone(tombstone),
2353 Timestamp::now(),
2354 )
2355 .await
2356 .unwrap();
2357
2358 let lt = InMemoryBackend::new("lt");
2359 let hv = TestBackend::with_inner(inner, CasConflict);
2360 let log = NoopChangeLog;
2361 let storage = TieredStorage::new(Box::new(hv), Box::new(lt), Box::new(log));
2362
2363 storage
2366 .put_object(
2367 &id,
2368 &Metadata::default(),
2369 stream::single("tiny"),
2370 Timestamp::now(),
2371 )
2372 .await
2373 .unwrap();
2374 }
2375
2376 #[derive(Debug)]
2380 struct FailCas(bool);
2381
2382 #[async_trait::async_trait]
2383 impl Hooks for FailCas {
2384 async fn compare_and_write(
2385 &self,
2386 inner: &InMemoryBackend,
2387 id: &ObjectId,
2388 current: Option<&ObjectId>,
2389 write: TieredWrite,
2390 access_time: Timestamp,
2391 ) -> Result<bool> {
2392 if self.0 {
2393 inner
2395 .compare_and_write(id, current, write, access_time)
2396 .await?;
2397 }
2398 Err(Error::with_source(
2399 ErrorKind::BackendFailure,
2400 std::io::Error::new(
2401 std::io::ErrorKind::TimedOut,
2402 "simulated compare_and_write failure",
2403 ),
2404 ))
2405 }
2406 }
2407
2408 #[tokio::test]
2412 async fn no_orphan_when_tombstone_write_fails() {
2413 let lt = InMemoryBackend::new("lt");
2414 let hv = TestBackend::new(FailCas(false));
2415 let log = NoopChangeLog;
2416 let storage = TieredStorage::new(Box::new(hv), Box::new(lt.clone()), Box::new(log));
2417
2418 let id = make_id("orphan-test");
2419 let payload = vec![0xABu8; 2 * 1024 * 1024]; let result = storage
2421 .put_object(
2422 &id,
2423 &Metadata::default(),
2424 stream::single(payload),
2425 Timestamp::now(),
2426 )
2427 .await;
2428
2429 assert!(result.is_err());
2430
2431 storage.join().await;
2433
2434 assert!(lt.is_empty(), "long-term object not cleaned up");
2435 }
2436
2437 #[tokio::test]
2438 async fn upload_cas_failure_cleans_up_new_blob() {
2439 let hv = TestBackend::new(FailCas(false));
2440 let lt = InMemoryBackend::new("lt");
2441 let storage =
2442 TieredStorage::new(Box::new(hv), Box::new(lt.clone()), Box::new(NoopChangeLog));
2443 let id = make_id("upload-cas-failure");
2444 let payload = vec![0xAB; BACKEND_SIZE_THRESHOLD + 1];
2445 let session =
2446 resumable_token(&storage, &id, &Metadata::default(), payload.len() as u64).await;
2447
2448 assert_eq!(
2449 storage
2450 .put_chunk(&session, 0, payload.len() as u64, stream::single(payload))
2451 .await
2452 .unwrap_err()
2453 .kind(),
2454 ErrorKind::BackendFailure
2455 );
2456 storage.join().await;
2457 assert!(
2458 lt.is_empty(),
2459 "LT blob should be cleaned up after CAS failure"
2460 );
2461 }
2462
2463 #[tokio::test]
2467 async fn orphan_tombstone_returns_none() {
2468 let (storage, hv, lt, _) = make_tiered_storage();
2469 let id = make_id("orphan-tombstone");
2470 let payload = vec![0xCDu8; 2 * 1024 * 1024]; storage
2473 .put_object(
2474 &id,
2475 &Metadata::default(),
2476 stream::single(payload),
2477 Timestamp::now(),
2478 )
2479 .await
2480 .unwrap();
2481
2482 let lt_id = hv.get(&id).expect_tombstone().target;
2484
2485 lt.remove(<_id);
2487
2488 assert!(
2489 storage
2490 .get_object(&id, Timestamp::now(), None)
2491 .await
2492 .unwrap()
2493 .is_none(),
2494 "orphan tombstone should resolve to None on get_object"
2495 );
2496 assert!(
2497 storage
2498 .get_metadata(&id, Timestamp::now())
2499 .await
2500 .unwrap()
2501 .is_none(),
2502 "orphan tombstone should resolve to None on get_metadata"
2503 );
2504 }
2505
2506 #[tokio::test]
2511 async fn tombstone_target_is_used_for_reads_and_deletes() {
2512 let hv = InMemoryBackend::new("hv");
2513 let lt = InMemoryBackend::new("lt");
2514 let log = NoopChangeLog;
2515 let storage = TieredStorage::new(Box::new(hv.clone()), Box::new(lt.clone()), Box::new(log));
2516
2517 let hv_id = make_id("hv-key");
2518 let lt_id = make_id("lt-key");
2519 let payload = vec![0xABu8; 100];
2520
2521 lt.put_object(
2523 <_id,
2524 &Metadata::default(),
2525 stream::single(payload.clone()),
2526 Timestamp::now(),
2527 )
2528 .await
2529 .unwrap();
2530 let tombstone = Tombstone {
2531 target: lt_id.clone(),
2532 time_expires: None,
2533 };
2534 hv.compare_and_write(
2535 &hv_id,
2536 None,
2537 TieredWrite::Tombstone(tombstone),
2538 Timestamp::now(),
2539 )
2540 .await
2541 .unwrap();
2542
2543 let (_, _, s) = storage
2545 .get_object(&hv_id, Timestamp::now(), None)
2546 .await
2547 .unwrap()
2548 .unwrap();
2549 let body = stream::read_to_vec(s).await.unwrap();
2550 assert_eq!(body, payload);
2551
2552 storage
2554 .delete_object(&hv_id, Timestamp::now())
2555 .await
2556 .unwrap();
2557 storage.join().await;
2558 assert!(!hv.contains(&hv_id), "tombstone should be removed");
2559 assert!(!lt.contains(<_id), "lt object should be removed");
2560 }
2561
2562 #[tokio::test]
2565 async fn multi_chunk_large_object_chains_buffered_and_remaining() {
2566 let (storage, hv, lt, _) = make_tiered_storage();
2567 let id = make_id("multi-chunk");
2568
2569 let chunk_size = 512 * 1024; let chunk_count = 4; let stream: ClientStream = futures_util::stream::iter(
2574 (0..chunk_count).map(move |i| Ok(Bytes::from(vec![i as u8; chunk_size]))),
2575 )
2576 .boxed();
2577
2578 storage
2579 .put_object(&id, &Metadata::default(), stream, Timestamp::now())
2580 .await
2581 .unwrap();
2582
2583 let lt_id = hv.get(&id).expect_tombstone().target;
2585 let (_, lt_bytes) = lt.get(<_id).expect_object();
2586 assert_eq!(lt_bytes.len(), chunk_size * chunk_count);
2587
2588 for i in 0..chunk_count {
2590 let offset = i * chunk_size;
2591 assert!(
2592 lt_bytes[offset..offset + chunk_size]
2593 .iter()
2594 .all(|&b| b == i as u8),
2595 "data mismatch in chunk {i}"
2596 );
2597 }
2598 }
2599
2600 #[tokio::test]
2606 async fn written_cleanup_after_lost_cas_response() {
2607 let (storage, hv, lt, log) = make_tiered_storage();
2608 let id = make_id("obj");
2609
2610 let payload = vec![0xAAu8; 2 * 1024 * 1024];
2612 storage
2613 .put_object(
2614 &id,
2615 &Metadata::default(),
2616 stream::single(payload.clone()),
2617 Timestamp::now(),
2618 )
2619 .await
2620 .unwrap();
2621 let tombstone1 = hv.get(&id).expect_tombstone().target;
2622
2623 let broken_storage = TieredStorage::new(
2625 Box::new(TestBackend::with_inner(hv.clone(), FailCas(true))),
2626 Box::new(lt.clone()),
2627 Box::new(log.clone()),
2628 );
2629 broken_storage
2630 .put_object(
2631 &id,
2632 &Metadata::default(),
2633 stream::single(payload.clone()),
2634 Timestamp::now(),
2635 )
2636 .await
2637 .unwrap_err(); let tombstone2 = hv.get(&id).expect_tombstone().target;
2639 assert_ne!(tombstone1, tombstone2);
2640
2641 broken_storage.join().await;
2643 lt.get(&tombstone1).expect_not_found();
2644 lt.get(&tombstone2).expect_object();
2645
2646 broken_storage
2648 .delete_object(&id, Timestamp::now())
2649 .await
2650 .unwrap_err();
2651 hv.get(&id).expect_not_found();
2652 broken_storage.join().await;
2653 lt.get(&tombstone2).expect_not_found();
2654
2655 let id = make_id("obj2");
2657 storage
2658 .put_object(
2659 &id,
2660 &Metadata::default(),
2661 stream::single(payload.clone()),
2662 Timestamp::now(),
2663 )
2664 .await
2665 .unwrap();
2666 let tombstone3 = hv.get(&id).expect_tombstone().target;
2667
2668 broken_storage
2670 .put_object(
2671 &id,
2672 &Metadata::default(),
2673 stream::single(&b"small"[..]),
2674 Timestamp::now(),
2675 )
2676 .await
2677 .unwrap_err(); hv.get(&id).expect_object();
2679 broken_storage.join().await;
2680 lt.get(&tombstone3).expect_not_found();
2681 }
2682
2683 #[test]
2687 fn guard_dropped_outside_runtime_does_not_panic() {
2688 let manager = ChangeManager::new(
2689 Box::new(InMemoryBackend::new("hv")),
2690 Box::new(InMemoryBackend::new("lt")),
2691 Box::new(NoopChangeLog),
2692 );
2693
2694 let change = Change {
2695 id: make_id("object-key"),
2696 new: Some(make_id("cleanup-target")),
2697 old: None,
2698 };
2699
2700 let guard = {
2703 let rt = tokio::runtime::Runtime::new().unwrap();
2704 rt.block_on(manager.record(change)).unwrap()
2705 };
2706
2707 drop(guard); }
2709
2710 #[tokio::test(start_paused = true)]
2715 async fn join_waits_for_cleanup_to_complete() {
2716 let (storage, _hv, _lt, _) = make_tiered_storage();
2717 let change = Change {
2718 id: make_id("object-key"),
2719 new: None,
2720 old: None,
2721 };
2722 let mut guard = storage.record_change(change).await.unwrap();
2723
2724 tokio::spawn(async move {
2725 tokio::time::sleep(Duration::from_secs(10)).await;
2726 guard.advance(ChangePhase::Completed);
2727 drop(guard);
2728 });
2729
2730 let join_future = tokio::spawn(async move { storage.join().await });
2731
2732 tokio::time::sleep(Duration::from_secs(9)).await;
2733 assert!(!join_future.is_finished(), "finished before guard dropped");
2734
2735 tokio::time::sleep(Duration::from_secs(2)).await;
2736 assert!(join_future.is_finished(), "finish after guard drops");
2737 }
2738
2739 #[derive(Clone, Debug)]
2746 struct PauseAfterPut {
2747 paused: Arc<tokio::sync::Notify>,
2748 resume: Arc<tokio::sync::Notify>,
2749 }
2750
2751 #[async_trait::async_trait]
2752 impl Hooks for PauseAfterPut {
2753 async fn put_object(
2754 &self,
2755 inner: &InMemoryBackend,
2756 id: &ObjectId,
2757 metadata: &Metadata,
2758 stream: ClientStream,
2759 access_time: Timestamp,
2760 ) -> Result<PutResponse> {
2761 inner.put_object(id, metadata, stream, access_time).await?;
2762 self.paused.notify_one();
2763 self.resume.notified().await;
2764 Ok(())
2765 }
2766 }
2767
2768 #[tokio::test]
2771 async fn dropped_future_triggers_cleanup_and_log_entry_removed() {
2772 let paused = Arc::new(tokio::sync::Notify::new());
2773 let hooks = PauseAfterPut {
2774 paused: Arc::clone(&paused),
2775 resume: Arc::new(tokio::sync::Notify::new()),
2776 };
2777
2778 let lt_inner = InMemoryBackend::new("lt");
2779 let log = InMemoryChangeLog::default();
2780 let storage = TieredStorage::new(
2781 Box::new(InMemoryBackend::new("hv")),
2782 Box::new(TestBackend::with_inner(lt_inner.clone(), hooks)),
2783 Box::new(log.clone()),
2784 );
2785
2786 let id = make_id("drop-test");
2787 let metadata = Metadata::default();
2788 let payload = vec![0xABu8; 2 * 1024 * 1024]; tokio::select! {
2792 result = storage.put_object(&id, &metadata, stream::single(payload), Timestamp::now()) => {
2793 panic!("expected put to pause before completing, got: {result:?}");
2794 }
2795 _ = paused.notified() => {
2796 }
2798 }
2799
2800 storage.join().await;
2802
2803 assert!(lt_inner.is_empty(), "orphaned LT blob was not cleaned up");
2805
2806 let entries = log.scan().await.unwrap();
2808 assert!(
2809 entries.is_empty(),
2810 "changelog entry not removed after cleanup"
2811 );
2812 }
2813}